diff --git a/.env.example b/.env.example index 2b32655..a57cdca 100644 --- a/.env.example +++ b/.env.example @@ -6,25 +6,65 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2): +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# SeaweedFS S3 (since lab4) +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url + +# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox); +# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher, +# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor. +BROKER_URL_FILE= WORKER_CONCURRENCY=2 +WORKER_PREFETCH=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=15 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=60 +WORKER_MEMORY_LIMIT_MB=512 STAGING_RETENTION_SECONDS=86400 +PIPELINE_VERSION=1 +# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the +# training stand by default; set FAULT_INJECTION=0 here to disable. + +# Pools and timeouts (same as lab3) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 + +# lab4: S3 client and uploads +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=5 +S3_MAX_ATTEMPTS=2 +S3_POOL_MAX=20 +UPLOAD_TIMEOUT_SECONDS=60 +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +DECODE_CONCURRENCY=2 +# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval +ATTACHMENT_GRACE_SECONDS=3600 +JANITOR_INTERVAL_SECONDS=300 + +# lab6: OpenTelemetry. Standard SDK variables; compose.yaml sets the same values for every app service +# and a distinct OTEL_SERVICE_NAME (messenger-api / messenger-worker / messenger-dispatcher / messenger-janitor). OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# true -> SDK off in all app processes (logs still go to stderr as JSON) +OTEL_SDK_DISABLED=false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c6d755c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf +*.yaml text eol=lf +*.yml text eol=lf +*.json text eol=lf +*.py text eol=lf +*.md text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..623eea0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бабиченко Максим, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесено всё решение lab5 (API, outbox, dispatcher, worker, janitor, тесты lab1–5). Добавлены: OpenTelemetry SDK во всех процессах (traces/metrics/logs по OTLP), W3C trace context API → outbox → RabbitMQ → worker (parent-child, replay — link), OTel Collector (memory_limiter, batch, ограниченные очереди), Prometheus (OTLP receiver, recording/alert rules + unit-тесты), Loki, Tempo, Grafana с provisioning источников и dashboard из Git, SLI/SLO, 6 алертов с runbooks, сценарии инцидентов. +Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`python scripts/init_secrets.py`, `docker compose up -d --build --wait`, `python student-tests/collect_evidence.py ...`). +Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев», `evidence/summary.md`, `evidence/incident-*.json`; прогон без Docker — `evidence/native-sandbox/`. +Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги» (одна копия каждого хранилища, буфер Collector в памяти, без TLS до lab7). +Источники и использование ИИ: REPORT.md, шапка (документация OTel/Prometheus/Loki/Tempo/Grafana; ИИ-ассистент Claude). -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [ ] make check и make test выполнены; приложен реальный вывод (Docker-прогон: `evidence/`). +- [ ] Сценарии отказов/безопасности своего уровня воспроизведены (Docker-прогон: `evidence/incident-*.json`). +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dc45121 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \ + pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py telemetry.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && mkdir -p /faults && chown appuser:appuser /faults +# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser +# копируется в пустой named volume при первом монтировании. + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index b97ae42..414451f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,368 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab5 (FastAPI, psycopg 3, pika, boto3, Pillow, RabbitMQ 4.1, SeaweedFS 3.80, PostgreSQL 17, Redis 7.4, HAProxy 3.2). Новое: OpenTelemetry Python SDK 1.45.2 (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) и `opentelemetry-instrumentation-psycopg` 0.66b2; образы `otel/opentelemetry-collector-contrib:0.164.0`, `prom/prometheus:v3.15.1`, `grafana/loki:3.7.9`, `grafana/tempo:3.1.1`, `grafana/grafana:12.4.14`, `busybox:1.37.0` (bootstrap прав volumes). Документация: OTel Collector configuration (memory_limiter, batch, exporterhelper sending_queue/retry_on_failure, internal telemetry), OTel semantic conventions (HTTP, database, messaging, `otel.sdk.*`), W3C Trace Context, Prometheus OTLP receiver и histograms/`histogram_quantile`, promtool unit testing, Loki OTLP ingestion и structured metadata, Tempo 3 single binary, Grafana provisioning (datasources, dashboards, derived fields, traces-to-logs). При написании кода, конфигураций и тестов использовался ИИ-ассистент (Claude); весь код я прогнал и могу объяснить и изменить на защите. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверяю на Windows (PowerShell, Docker Desktop). Python на хосте нужен только для тестов (стандартная библиотека). Перед запуском остановить стенд lab5 (`docker compose down` в его папке): заняты порты 8080/8404. Имя проекта Compose — `messenger-lab6`, volumes отдельные от lab5. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab6.git +cd mtusi2026lab6 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" +Copy-Item .env.example .env +python scripts/init_secrets.py # как в lab5 + secrets/grafana_admin_password (папка secrets/ в .gitignore) +docker compose --env-file .env config --quiet +docker compose up -d --build --wait # postgres, s3, rabbitmq, otel-collector, prometheus, loki, tempo -> + # migrate, s3-init, telemetry-init -> api x2, janitor, dispatcher, worker, + # grafana -> lb +docker compose ps -a # migrate, s3-init, telemetry-init: Exited (0); остальные running/healthy +python tests/smoke.py --base-url http://127.0.0.1:8080 # make test при включённой телеметрии + +# доказательства lab6 (~35 мин) в evidence/ и таблица замеров в этот отчёт: +python student-tests/collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history ` + incident-worker incident-db incident-telemetry incident-collector load +``` + +| Что | Где | +| --- | --- | +| Grafana (dashboard «Messenger — lab6 observability», Explore, Alerting) | http://127.0.0.1:3000, логин `admin`, пароль — `secrets/grafana_admin_password` | +| Prometheus: алерты pending/firing, запросы | http://127.0.0.1:9090/alerts, http://127.0.0.1:9090/graph | +| Loki / Tempo HTTP API (для тестов) | http://127.0.0.1:3100, http://127.0.0.1:3200 | +| Трасса конкретного ответа API | заголовок ответа `X-Trace-Id` (или свой `traceparent` в запросе) → Grafana → Explore → Tempo | + +Отдельные проверки (make на Windows нет): + +| Команда | Что проверяет | +| --- | --- | +| `python scripts/check_contract.py` | make check | +| `python student-tests/check_configs.py` | конфиги Collector/Prometheus/Loki/Tempo их же инструментами, unit-тесты алертов (promtool), provisioning Grafana через её API | +| `python student-tests/test_telemetry.py [request upload-chain retry-chain replay-link history]` | запрос ↔ лог ↔ трасса, цепочка upload → worker, retry, replay с link, сохранность истории | +| `python student-tests/check_dashboard.py` | каждый запрос dashboard и правил выполняется на живых данных, запрещённых меток нет | +| `python student-tests/incidents.py [worker-stop db-slow telemetry-down collector-down]` | инциденты: firing → диагностика → resolved | +| `docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml` | unit-тесты порогов/окон/`for` алертов | +| `python scripts/build_dashboard.py` | пересобрать `infra/grafana/dashboards/messenger.json` из описания панелей | + +Bootstrap: как в lab5 (`migrate`, `s3-init`, definitions RabbitMQ) плюс `telemetry-init` (одноразовый `chown 10001` volumes Loki и Tempo: оба процесса работают не от root). Provisioning Grafana — файлы `infra/grafana/provisioning` и `infra/grafana/dashboards`, монтируются только для чтения: источники данных и dashboard появляются при старте, ручного набора нет. Всё идемпотентно: повторный `up` ничего не пересоздаёт и не дублирует. + +Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults`, `promdata`, `lokidata`, `tempodata`, `grafanadata` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py`. Выключить телеметрию целиком (для сравнения): `OTEL_SDK_DISABLED=true` в `.env` и `docker compose up -d`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text + +--------------------------- сеть telemetry (internal) ----------------------------+ +Клиент -> lb -> api x2 ---|-- OTLP/HTTP :4318 --> otel-collector --+-- traces OTLP gRPC --> tempo (tempodata) | + | | (traces, metrics, memory_limiter |-- logs OTLP HTTP --> loki (lokidata) | + | | logs) batch, queues +-- metrics OTLP HTTP --> prometheus(promdata)| + PostgreSQL <-+ outbox | ^ pull :8888 (метрики Collector)| + (источник истины) | dispatcher, worker, janitor -- OTLP --^ | | + | grafana -------+--> prometheus, loki, tempo | + +----------------------------------------------------------------------------------+ + W3C traceparent: API (спан create, outbox) -> dispatcher (спан publish, AMQP headers + поле сообщения) + -> worker (спан process, consumer) -> S3/PostgreSQL; retry/replay -> новая строка outbox + 127.0.0.1: 8080 (lb), 8404 (stats), 3000 (grafana), 9090 (prometheus), 3100 (loki), 3200 (tempo) +``` + +Перенесено из lab5 без изменения бизнес-логики и контракта: всё приложение (API, outbox, dispatcher, worker, janitor, варианты изображений, повторы, DLQ, replay), миграции, тесты lab1–5, hooks. Бизнес-API не менялся; единственное расширение — необязательный заголовок ответа `X-Trace-Id` (trace_id запроса) и приём входящего `traceparent` (обратно совместимо, клиент может не знать о них). + +Что добавлено в lab6: +- `telemetry.py` — настройка OpenTelemetry SDK для всех процессов: ресурс, три провайдера (traces/metrics/logs) с OTLP/HTTP exporter, ограниченные очереди, сэмплер, JSON-логи в stderr + OTLP, метрики процесса/контейнера, W3C propagator, помощники спанов. +- Инструментирование: HTTP server-спаны и метрики в middleware `main.py`; спаны PostgreSQL (`opentelemetry-instrumentation-psycopg`, текст SQL с плейсхолдерами, без значений), S3 (`storage.py`), Redis и Argon2id (`auth.py`); messaging-спаны и перенос контекста через outbox/RabbitMQ (`pipeline.py`, `dispatcher.py`, `broker.py`, `worker.py`); метрики очереди/outbox (dispatcher), пула БД (`db.py`), занятости worker, времени до ready; структурные события логов (`event=...`). +- Конфигурации в Git: `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml`, `infra/prometheus/rules/{recording,alerts}.yml`, `infra/prometheus/tests/alerts_test.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, `infra/grafana/provisioning/**`, `infra/grafana/dashboards/messenger.json` (генерирует `scripts/build_dashboard.py`), `docs/runbooks.md`. +- `compose.yaml`: сервисы `otel-collector`, `telemetry-init`, `prometheus`, `loki`, `tempo`, `grafana`; сети `telemetry` (internal) и `ops` (только чтобы опубликовать UI на 127.0.0.1); volumes `promdata`, `lokidata`, `tempodata`, `grafanadata`; секрет `grafana_admin_password`. Restart hook перезапускает и хранилища телеметрии. +- Тесты: `student-tests/{obs.py, test_telemetry.py, check_dashboard.py, check_configs.py, incidents.py}`, обновлены `check_stack.py`, `run_persistence.py`, `collect_evidence.py`. + +### Роли и назначение хранилищ + +| Роль | Сервис | Версия | Зачем | Health | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Collector | `otel-collector` | otelcol-contrib 0.164.0 | приём OTLP, защита памяти, пакетирование, буфер на время недоступности хранилища, маршрутизация в 3 хранилища. **Не хранит** | метрики `otelcol_*` (scrape) — образ distroless, без shell | нет (очередь в памяти) | telemetry; 4317/4318/8888 не публикуются | нет | +| Метрики | `prometheus` | 3.15.1 | числовые ряды: «что, сколько, когда», percentiles из histogram, SLI, alert rules, состояние алертов | `/-/ready` | `promdata` (7 дней, ≤ 1 GB) | telemetry, ops; 127.0.0.1:9090 | нет | +| Логи | `loki` | 3.7.9 | события с контекстом: «почему» (error_type, request_id, trace_id) | — (`/ready` проверяют тесты) | `lokidata` (7 дней) | telemetry, ops; 127.0.0.1:3100 | нет | +| Трассы | `tempo` | 3.1.1 | причинная цепочка запроса через процессы и время каждого шага: «где» | — (`/ready`) | `tempodata` (7 дней) | telemetry, ops; 127.0.0.1:3200 | нет | +| UI | `grafana` | 12.4.14 | запросы к трём хранилищам, dashboard, связки лог ↔ трасса ↔ метрика; сама сигналы не хранит | `/api/health` | `grafanadata` (сессии/настройки UI) | telemetry, ops; 127.0.0.1:3000 | `grafana_admin_password` | + +Почему так: Grafana сама не хранит сигналы, а каждому сигналу нужно своё хранилище с подходящей моделью — TSDB для рядов (Prometheus умеет OTLP напрямую, поэтому без remote-write и без exporter-а в Collector), индекс по меткам + сжатые строки для логов (Loki принимает OTLP и хранит атрибуты записи как structured metadata, поэтому `trace_id`/`request_id` не становятся метками индекса), блоки трасс по trace_id (Tempo 3 single binary без Kafka). Collector отделяет приложение от хранилищ: приложение знает один адрес, а смена/недоступность хранилища его не касается. + +### Сигналы: контракт → реальные имена + +Имена OTel переводит в Prometheus `translation_strategy: UnderscoreEscapingWithSuffixes` (зафиксировано в `prometheus.yml`): точки → `_`, суффикс единицы (`_seconds`, `_bytes`) и `_total` для счётчиков; `service.name` → метка `job`, `service.instance.id` → `instance`, остальные ресурсные атрибуты — в `target_info`. + +| Сигнал контракта (уровень) | Инструмент OTel (единица) | В хранилище | Метки / поля | +| --- | --- | --- | --- | +| HTTP requests (3) | `http.server.request.duration` histogram, его `_count` | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`, `http_response_status_class` (`2xx`…) | +| HTTP duration (3) | то же, секунды; buckets 5 мс … 10 с | `http_server_request_duration_seconds_bucket` | те же + `le` | +| Jobs processed / failed (3) | `messenger.job.duration` histogram, `_count` (одна метрика, без дубля) | `messenger_job_duration_seconds_count` | `job_type="attachment.process.v1"`, `outcome` = ready / retry / failed / stale / lease_lost / db_error | +| Job processing duration (3) | `messenger.job.duration` (s), buckets 0,1 … 120 с | `messenger_job_duration_seconds_bucket` | те же | +| Доставки/публикации (доп.) | `messenger.messages.consumed`, `messenger.messages.published` counters | `messenger_messages_consumed_total`, `..._published_total` | `messaging_destination_name`, `outcome` (processed/noop/dead_lettered/requeued; ok/error) | +| Логи API и worker (3) | Python `logging` → OTel Logs API | Loki: метки `service_name`, `service_instance_id`, `deployment_environment_name`; structured metadata `event`, `severity_text`, `trace_id`, `span_id`, `request_id`, `job_id`, `error_type`… | в stderr та же запись одной JSON-строкой (`ts`, `severity`, `service.name`, `event`, `trace_id`, `span_id`) | +| Трассы API и worker (3) | server / producer / consumer / client / internal спаны | Tempo | `http.route`, `http.response.status_code`, `messaging.*`, `messenger.job.*`, `db.statement`, `aws.s3.key`, `error.type` | +| Queue depth / oldest job age (4) | `messenger.queue.depth` gauge; `messenger.queue.oldest_age` (s); доп. `messenger.jobs.backlog`, `messenger.queue.consumers`, `messenger.outbox.pending`, `messenger.outbox.oldest_age` (s) | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds`, … | `messaging_destination_name` (очередь/DLQ), `status` | +| DB pool used/waiting (4) | `db.client.connection.count` (`state`=used/idle), `.max`, `.pending_requests`, `.timeouts`, `db.client.connection.wait_time` (s, histogram) | `db_client_connection_*` | `db_client_connection_pool_name="messenger-api"`, `db_client_connection_state` | +| Worker busy/capacity (4) | `messenger.worker.busy`, `messenger.worker.capacity` | `messenger_worker_busy`, `messenger_worker_capacity` | `messaging_destination_name` | +| Process / container CPU, RAM (4) | `process.cpu.time` (s), `process.memory.usage` (By); `container.cpu.time`, `container.cpu.limit`, `container.memory.usage`, `container.memory.limit` (cgroup v2, только в Docker) | `process_cpu_time_seconds_total`, `process_memory_usage_bytes`, `container_*` | `cpu_mode` | +| End-to-end time to ready (4) | `messenger.attachment.time_to_ready` (s), buckets 0,5 … 600 с; от COMMIT задания в API до COMMIT ready, по часам PostgreSQL | `messenger_attachment_time_to_ready_seconds_*` | `job_type` | +| Сквозная связь upload → publish → consume → S3/DB (4) | W3C `traceparent`/`tracestate`: parent-child, replay — link | Tempo | см. ниже | +| Потери телеметрии (4) | SDK: `otel.sdk.processor.span.processed{error.type}`, `otel.sdk.exporter.span.exported{error.type}`; Collector: `otelcol_exporter_{sent,send_failed,enqueue_failed}_*`, `otelcol_exporter_queue_size/capacity`, `otelcol_receiver_refused_*` | Prometheus | `exporter`, `error_type` | + +Ресурс (одинаков для всех трёх сигналов процесса): `service.name` = `messenger-api` / `messenger-worker` / `messenger-dispatcher` / `messenger-janitor` (и `messenger-ops` у `ops.py`) — задаётся в коде процесса, поэтому общий `.env` не делает janitor «api»; `service.version` = `APP_VERSION`; `service.instance.id` = hostname контейнера (тот же, что `X-Instance-Id`); `deployment.environment.name=lab` (`OTEL_RESOURCE_ATTRIBUTES`). + +**Что не попадает в телеметрию.** Метки метрик — только перечисленные выше (метод, шаблон маршрута, класс/код статуса, тип/исход задания, очередь, состояние соединения, режим CPU); `check_dashboard.py` проверяет, что у метрик приложения нет `user_id`, `chat_id`, `message_id`, `attachment_id`, `request_id`, URL, имени файла, текста и значений-UUID. В спанах и логах есть технические ID синтетических данных (`job_id`, `attachment_id`, `request_id`, ключ S3 вида `attachments//v1/crop.jpg`) — без них нельзя пройти от загрузки к результату worker. Не пишутся: пароли, `Authorization`, токен сессии и его хеш (спаны Redis без ключа), текст сообщения, имя файла, байты изображения, значения параметров SQL (`capture_parameters=False`), signed URL (их нет). `test_telemetry.py request` ищет текст сообщения, пароль и токен в Loki и атрибутах спанов — 0 совпадений. В Collector дополнительно `attributes/scrub` удаляет любой атрибут, чьё имя похоже на password/authorization/cookie/secret/token/api key. + +### Перенос контекста через очередь + +```text +[api] POST /api/v1/chats/{chat_id}/attachments SERVER + ├─ PostgreSQL INSERT attachments, S3 PutObject staging/, UPDATE ... state='queued' + └─ create attachments PRODUCER ← строка outbox в ТОЙ ЖЕ транзакции; traceparent этого спана + │ пишется в attachment_jobs.traceparent и в payload outbox + └─ [dispatcher] publish attachments PRODUCER ← родитель = traceparent из outbox; свой контекст + │ кладёт в AMQP header traceparent и в поле сообщения + └─ [worker] process attachments.process.v1 CONSUMER ← родитель = header (запасной — поле тела) + ├─ claim (PostgreSQL), S3 GetObject, imaging, S3 PutObject x4, commit result (PostgreSQL) + ├─ (ошибка) create attachments PRODUCER → publish → process (попытка 2) — та же трасса + └─ cleanup staging (S3 DeleteObject, UPDATE) — после ack, в той же трассе +``` + +Выбранная семантика: **parent-child** вдоль причинной цепочки (публикация и обработка — продолжение запроса пользователя, а задержка между ними видна как разрыв во времени). **Link** — когда это новое решение, а не продолжение: `ops.py replay` создаёт новую трассу (корень `ops replay`), спан `create` которой ссылается на исходную загрузку. Восстановление истёкшей аренды dispatcher-ом и сверка «queued без сообщения» создают спаны `recover expired lease` / `republish lost message` с родителем из `attachment_jobs.traceparent`. В lab5 `traceparent` был случайным; теперь это контекст реального спана. + +Сэмплирование: `ParentBased(AlwaysOn)`, но клиентский спан без родителя не записывается — это фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, проверка соединений пула). Всё, что относится к запросу или заданию, сохраняется полностью (100%). Проверки здоровья (`/health/*`, HAProxy раз в секунду) — только в метриках, без трасс. + +### SLI / SLO и окно + +| SLO | SLI (числитель / знаменатель) | Цель | Окно | Recording rule | +| --- | --- | --- | --- | --- | +| SLO-1 доступность API | бизнес-запросы API без 5xx / все бизнес-запросы API (все маршруты кроме `/health/*`). 4xx — ошибки клиента, не входят в числитель | ≥ 99% | скользящий 1 ч (учебное окно; в эксплуатации то же выражение с 28d) | `messenger:slo_availability:ratio_1h`, бюджет `messenger:slo_availability:error_budget_remaining_1h` | +| SLO-2 задержка записи | успешные `POST /api/v1/chats/{chat_id}/messages` быстрее 250 мс (bucket `le="0.25"`) / все успешные такие запросы | ≥ 95% | 1 ч | `messenger:slo_latency_messages:ratio_1h` | +| SLO-3 фоновая обработка | вложения ready не позже 30 с от приёма (bucket `le="30"` гистограммы time_to_ready) / все ставшие ready | ≥ 95% | 1 ч | `messenger:slo_time_to_ready:ratio_1h` | + +Проверяемость: все SLI — отношения счётчиков/buckets гистограмм, просуммированных по репликам (`sum(increase(...[1h]))`), поэтому SLO вычисляется одинаково на любом наборе реплик и видно на dashboard (строка «Обзор и SLO»). + +### Dashboard: запросы и единицы + +Dashboard — `infra/grafana/dashboards/messenger.json` (импортируется provisioning-ом, папка Messenger; 34 панели в 6 строках: обзор/SLO и алерты, API RED, фоновая обработка, насыщение, конвейер телеметрии, логи и трассы). Ключевые запросы (`$__rate_interval` подставляет Grafana; `API` = `job="messenger-api", http_route!~"/health/.*"`): + +| Панель | PromQL | Единица | +| --- | --- | --- | +| RPS по маршрутам / по классу статуса | `sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{API}[$__rate_interval]))` | req/s | +| Доля 5xx | `(sum(rate(..._count{API, http_response_status_class="5xx"}[…])) or on() vector(0)) / sum(rate(..._count{API}[…]))` | доля (0–1) | +| p50 / p95 / p99 API | `histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{API}[$__rate_interval])))` | с | +| p95 по маршрутам | `histogram_quantile(0.95, sum by (le, http_route) (rate(..._bucket{API}[…])))` | с | +| Попытки заданий по исходу | `sum by (outcome) (rate(messenger_job_duration_seconds_count[…]))` | 1/с | +| Длительность попытки / время до ready p50/p95/p99 | `histogram_quantile(q, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="ready"}[…])))`, то же для `messenger_attachment_time_to_ready_seconds_bucket` | с | +| Глубина очереди и backlog | `max by (messaging_destination_name) (messenger_queue_depth)`, `max by (status) (messenger_jobs_backlog)` | сообщений / заданий | +| Возраст старейшего задания и outbox | `max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})`, `max(messenger_outbox_oldest_age_seconds)` | с | +| Пул БД | `sum by (instance) (db_client_connection_count{db_client_connection_state="used"})`, `max(db_client_connection_max)`, `sum by (instance) (db_client_connection_pending_requests)`; p95 ожидания `histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[…])))` | соединений / с | +| Насыщение worker | `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` | доля | +| CPU / RAM | `sum by (job, instance) (rate(process_cpu_time_seconds_total[…]))` (ядер), `rate(container_cpu_time_seconds_total) / container_cpu_limit` (доля лимита), `process_memory_usage_bytes`, `container_memory_usage_bytes` | ядра / доля / байты | +| Collector и SDK | `rate(otelcol_receiver_accepted_spans[…])`, `rate(otelcol_exporter_send_failed_spans[…])`, `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`, `otelcol_process_memory_rss`, `rate(otel_sdk_exporter_span_exported_total{error_type!=""}[…])` | 1/с, доля, байты | +| Логи / трассы | Loki `{service_name=~"messenger-.*"} \| detected_level=~"warn\|error"`; Tempo TraceQL `{ status = error }` | — | + +Percentile только из buckets гистограммы, просуммированных по всем репликам **до** `histogram_quantile` (`sum by (le)`); среднее и усреднение p95 реплик не используются. Точность — ширина bucket: например, при запросах по 5,0 с (statement_timeout) интерполяция внутри bucket 5–7,5 с даёт p95 ≈ 7,4 с; это ограничение метода, видно на инциденте 2. + +### Алерты + +`infra/prometheus/rules/alerts.yml`; состояние — Prometheus `/alerts` и `ALERTS{}` (видно и resolved по истории), Grafana показывает их как data source rules. Внешней доставки нет (по заданию). + +| Алерт | Условие | Окно | Порог | for | Runbook | +| --- | --- | --- | --- | --- | --- | +| MessengerHighErrorRate | доля 5xx бизнес-запросов API и трафик > 0,2 запроса/с | rate 1m | > 5% (в 5 раз выше бюджета SLO-1) | 1m | `docs/runbooks.md#messengerhigherrorrate` | +| MessengerQueueLagGrowing | возраст старейшего готового задания | мгновенное значение (обновление 5–10 с) | > 30 с (SLO-3) | 1m | `#messengerqueuelaggrowing` | +| MessengerWorkerNoConsumers | consumers очереди = 0 | — | 0 | 1m | `#messengerworkernoconsumers` | +| MessengerDbPoolSaturated | запросы ждут соединение из пула | — | > 0 | 1m | `#messengerdbpoolsaturated` | +| TelemetryExportFailing | потери в Collector (send/enqueue failed) **или** очередь exporter заполнена > 5% | rate 2m | > 0 / > 5% | 1m | `#telemetryexportfailing` | +| TelemetryCollectorDown | `up{job="otel-collector"} == 0` | — | — | 1m | `#telemetrycollectordown` | + +Пороги, окна и `for` проверены unit-тестами `infra/prometheus/tests/alerts_test.yml` (promtool: не срабатывает раньше `for`, не срабатывает при пустом трафике и на ошибках `/health`, resolved после восстановления) и живыми инцидентами ниже. + +### Collector и SDK: память и недоступность хранилища + +- SDK (каждый процесс): `BatchSpanProcessor` и `BatchLogRecordProcessor` с очередью 2048 (`OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_QUEUE_SIZE`), экспорт в фоновом потоке, таймаут 5 с (`OTEL_EXPORTER_OTLP_TIMEOUT`); метрики — агрегаты фиксированного размера раз в 10 с. Переполнение и непринятые пакеты считаются самим SDK (`OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED=true`). Ошибки экспорта пишутся в лог не чаще раза в 30 с и не отправляются тем же экспортом (иначе петля). +- Collector: первым в каждом pipeline `memory_limiter` (heap Go: жёсткий предел 200 MiB, мягкий 160 MiB — receiver отвечает клиентам ошибкой, SDK отбрасывает у себя), затем `batch` (2 с / 1024–2048 элементов). У каждого exporter своя `sending_queue` в памяти (Tempo, Loki — 200 пакетов, Prometheus — 100) и `retry_on_failure` (1 → 10 с, `max_elapsed_time` 120 с). Окно повторов считается для пакета с первой попытки; ожидание в очереди в него не входит, поэтому буфер на время недоступности — это очередь. Пакет отбрасывается, когда истекли его 120 с повторов (`otelcol_exporter_send_failed_*`) или очередь полна (`otelcol_exporter_enqueue_failed_*`). Pipeline независимы: недоступный Tempo не мешает логам и метрикам. Очередь не персистентная: рестарт Collector теряет буфер (осознанный компромисс учебного стенда, `file_storage` не нужен). +- Контейнер Collector 384 MiB: RSS = heap (≤ 200 MiB) + runtime Go + отображённый в память бинарник contrib (~150 MiB вытесняемых страниц); поэтому тест сравнивает с лимитом именно heap (`otelcol_process_runtime_heap_alloc_bytes`), а RSS/память контейнера приводит для сведения. + +### Соответствие `.env.example` + +| Переменная | Что делает в этом решении | +| --- | --- | +| всё из lab5 | как в lab5 (см. REPORT lab5) | +| OTEL_SERVICE_NAME | `messenger-api` для api; compose задаёт свой каждому сервису; фактически `service.name` берётся из кода процесса | +| OTEL_EXPORTER_OTLP_ENDPOINT / _PROTOCOL | `http://otel-collector:4318`, `http/protobuf` (SDK добавляет `/v1/traces`, `/v1/metrics`, `/v1/logs`) | +| OTEL_RESOURCE_ATTRIBUTES | `deployment.environment.name=lab` | +| OTEL_SDK_DISABLED | `true` — SDK выключен во всех процессах (логи остаются JSON в stderr); для сравнения telemetry off/on | +| (только compose) OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_METRIC_EXPORT_INTERVAL, OTEL_BSP_MAX_QUEUE_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED | 5 с, 10000 мс, 2048, 2048, `true` | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +`python student-tests/collect_evidence.py …` пишет вывод каждого шага в `evidence/<шаг>.txt` (дата, commit, код выхода), итог — `evidence/summary.md`, JSON инцидентов — `evidence/incident-*.json`, а таблицы замеров — в блок «Замеры (Docker)» ниже. Те же сценарии прогнаны в нативной песочнице (те же версии Collector/Prometheus/Loki/Tempo, процессы без Docker; см. «Где проверено») — `evidence/native-sandbox/`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker-стенд, `evidence/`; то же в нативной песочнице — `evidence/native-sandbox/`) | Файл доказательства | +| --- | --- | --- | --- | +| 3: минимум lab5 работает, `make test` при включённой телеметрии | `tests/smoke.py` | PASS через балансировщик с SDK во всех процессах | smoke.txt | +| 3: Collector, 3 хранилища и Grafana в одном воспроизводимом стенде; конфиги/provisioning в Git; persistent volumes | `check_stack.py`, `check_configs.py`, `test_telemetry.py history` | конфиги валидны инструментами тех же версий; unit-тесты алертов SUCCESS; после restart prometheus/loki/tempo прежние трасса, логи и точка метрики находятся | stack.txt, configs.txt, telemetry-history.txt | +| 3: API и worker отдают OTLP; RPS/errors/duration API, jobs ok/failed и duration worker; логи и реальные трассы | `check_dashboard.py`, `test_telemetry.py` | все 54 PromQL-выражения dashboard и 16 выражений правил выполняются без ошибок, ключевые сигналы с данными; трассы API, dispatcher, worker | dashboard.txt, telemetry.txt | +| 3: импортируемый dashboard; низкая кардинальность; trace/span id в логах; нет содержимого/секретов | `check_dashboard.py`, `check_configs.py` (Grafana API), `test_telemetry.py request` | метки только из белого списка, `http_route` — шаблоны; dashboard provisioned=true; текст, пароль, токен: 0 совпадений в Loki и спанах | dashboard.txt, configs.txt, telemetry.txt | +| 3: контролируемый запрос → trace + лог; job → worker trace + лог | `test_telemetry.py request upload-chain` | свой `traceparent` → server-спан с дочерними PostgreSQL/Redis → лог `event=http.request` с тем же trace_id/span_id и request_id = `X-Request-Id`; логи worker `job.claimed`/`job.ready` с trace_id загрузки | telemetry.txt | +| 3: назначение каждого хранилища | раздел «Роли и назначение хранилищ» | — | REPORT.md | +| 4: W3C через API → broker → worker, producer/consumer, цепочка до S3/DB, включая retry | `test_telemetry.py upload-chain retry-chain replay-link` | одна трасса: server → create (producer) → publish (producer, dispatcher) → process (consumer, worker) → 4×S3 PutObject + commit; retry: попытка 1 ERROR `storage_unavailable` → create → publish → попытка 2 ready; replay — новая трасса с link на исходную | telemetry.txt | +| 4: p50/p95/p99 из histogram, доля ошибок, queue depth/oldest age, DB pool, saturation worker; запросы и единицы | dashboard + раздел «Dashboard: запросы и единицы» | все панели выполняются на живых данных | dashboard.txt | +| 4: SLI/SLO и окно; alert rules high error rate и queue lag; проверены отказом (firing/resolved) | `incidents.py worker-stop db-slow`, `promtool test rules` | см. таблицы инцидентов: firing и resolved по каждому алерту | incident-worker.txt, incident-db.txt, configs.txt | +| 4: memory limiter/batch, ограниченная буферизация; остановка хранилища телеметрии не ломает сообщения и не растит RAM | `incidents.py telemetry-down collector-down` | API 100% успешных ответов, heap Collector ограничен, потери посчитаны | incident-telemetry.txt, incident-collector.txt | +| 4: разбор двух инцидентов и шаги диагностики | раздел «Эксперименты и отказы» | — | REPORT.md, docs/runbooks.md | +| Выбранный трек 5 | — | не выбран | — | + +### Где проверено + +**Нативная песочница (выполнено).** Среда, где писалось решение, не имеет доступа к Docker Hub, поэтому весь стенд был поднят теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API за балансировщиком, dispatcher, worker, janitor (Python 3.11, те же `requirements.txt`) и **те же версии** otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из `infra/` (заменены только адреса `tempo:4317` → `127.0.0.1:14317` и т.п.). Тесты запускались без изменений через тонкую подмену команды `docker compose` (stop/start/exec на процессы). Результаты — `evidence/native-sandbox/`, цифры — в разделе ниже. Grafana в песочнице не запускалась (нет бинарника без Docker Hub): её provisioning проверяет `check_configs.py` через HTTP API Grafana на Docker-стенде, а все запросы dashboard выполнены против живых Prometheus/Loki/Tempo (`check_dashboard.py`). + +**Docker-стенд (Windows 10, Docker Desktop 28.1.1, Compose 2.35.1; VM 16 CPU / 15,2 GiB) — выполнено 2026-11-01.** Стенд поднят из чистого состояния (`init_secrets.py`, `docker compose pull`, `up -d --build --wait`) без правок кода и конфигов: `migrate`, `s3-init`, `telemetry-init` — Exited (0), остальные running/healthy; Loki и Tempo пишут в volumes от uid 10001 без ошибок прав. Полный прогон `collect_evidence.py` — все шаги PASS: lab6 (check, stack, smoke, configs, telemetry, dashboard, telemetry-history, 4 инцидента, load) и регрессия lab1–5 (`--from jobs-variants`, 22 шага, включая failover и persistence; restart hook вместе с ожиданием готовности — 60,9 с при лимите 90 с). Вывод — `evidence/*.txt`, сводка — `evidence/summary.md`, таблицы — блок «Замеры (Docker)» ниже. Grafana проверена через её API: 3 источника данных healthy, dashboard provisioned, трасса по `X-Trace-Id` открывается через источник Tempo, лог с тем же `trace_id` — через Loki (связки tracesToLogs и derived field TraceID). ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. По сохранённым метрикам, логам и трассам можно найти причину инцидента, не заходя в контейнеры: остановленный worker отличается от медленной БД, а медленная БД — от медленного обработчика. +2. Недоступность любого звена телеметрии (хранилище, Collector) не влияет на ответы API, память Collector и процессов приложения ограничена, а потери телеметрии считаются. + +Методика: `student-tests/incidents.py` — нагрузка (closed loop, POST сообщений), контролируемый отказ, опрос `/api/v1/alerts` каждые 5 с до firing, «диагностика» только запросами к Prometheus/Loki/Tempo, устранение отказа, ожидание resolved; хронология в выводе, итог в `evidence/incident-*.json` (без токенов и текстов). + +### Нативная песочница: результаты + +Окружение: Linux x86_64, 2 vCPU, 8 GiB RAM на всё (стенд, хранилища телеметрии и клиент нагрузки); PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, Python 3.11; otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 (те же версии и конфиги, что в compose.yaml). Дата прогона 2026-11-01. Сводка шагов (все PASS, включая регрессию lab1–5: варианты изображений, outbox, повторы/DLQ/replay, конкуренция worker, вложения, идемпотентность, гонки) — `evidence/native-sandbox/summary.md`. + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с | +| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) | + +Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной). + +### Инцидент 1: остановлен worker («задания не обрабатываются») + +Воздействие: `docker compose stop worker`, затем 3 загрузки вложений (API отвечает 202 queued — приём от worker не зависит). + +Шаги диагностики (как в `docs/runbooks.md`): +1. **Алерт** `MessengerWorkerNoConsumers` (consumers = 0), затем `MessengerQueueLagGrowing` (возраст старейшего задания > 30 с). Dashboard «Возраст старейшего задания» растёт линейно — задания не забирают, а не «их стало много». +2. **Публикация или обработка?** `messenger_outbox_oldest_age_seconds` = 0 и `messenger_messages_published_total{outcome="ok"}` растёт → dispatcher и RabbitMQ работают, сообщения в брокере (`messenger_queue_depth` = числу загрузок). +3. **Кто не забирает?** `messenger_queue_consumers` = 0, метрики `messenger_worker_*` перестали обновляться (worker не экспортирует), последний лог worker в Loki — `event=service.stopped` (штатная остановка, а не падение — тогда был бы `job.handler_crashed`/обрыв без stopped). +4. **Трасса любой из загрузок** (trace_id из лога API `event=http.request` или `X-Trace-Id`): есть `publish attachments`, нет `process attachments.process.v1` — сообщение доставлено в очередь, но не взято. +5. Причина: worker остановлен. Действие: `docker compose start worker`. Проверка: вложения ready через 2–5 с (Docker 4,7 с, песочница 2,0 с), backlog → 0, оба алерта resolved (Docker — через 29,7 с после start). + +### Инцидент 2: медленная БД (блокировка таблицы `messages`) vs медленный обработчик + +Воздействие: под нагрузкой 8 потоков `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` в отдельной транзакции на 210 с (PostgreSQL жив, `/health/ready` 200, но каждая запись ждёт блокировку и через `statement_timeout` 5 с отменяется → API 503). + +Шаги диагностики: +1. **Алерт** `MessengerHighErrorRate`. Срабатывает не сразу: окно rate 1m заполнено быстрыми запросами до инцидента (~115–130/с), а во время блокировки closed-loop нагрузка даёт ~1,6 запроса/с (каждый висит 5 с), поэтому доля > 5% появляется только через 50–80 с, плюс `for: 1m` (firing через 140,8 с от LOCK в Docker, 120,1 с в песочнице). +2. **Какие маршруты и коды**: только `POST /api/v1/chats/{chat_id}/messages`, код 503; чтения других таблиц (`GET /api/v1/users/me`, `GET /api/v1/chats`) во время блокировки 5–20 мс (Docker 17,2 мс, песочница 4,7 мс) → не «всё медленно», а конкретная таблица/операция. +3. **БД или обработчик — по трассе**: TraceQL `{ resource.service.name = "messenger-api" && status = error }` → server-спан ≈ 5005–5015 мс, из них дочерний `INSERT` (PostgreSQL) ≈ 5001 мс со статусом ERROR — 99,8–99,9% времени запрос ждал БД. Для контраста медленный **обработчик**: `POST /api/v1/users` — server 20–35 мс, из них `argon2id hash` 16–29 мс (CPU в процессе API), все спаны БД < 1 мс. По метрикам то же различие: при медленной БД растут `db_client_connection_count{state="used"}` (соединения заняты ожиданием) и p95 конкретного маршрута, при медленном обработчике пул свободен, а растёт CPU процесса API (`process_cpu_time_seconds_total`). +4. **Почему — по логам**: `{service_name="messenger-api"} | event="db.unavailable"` с `error_type=QueryCanceled` (statement_timeout), а не `OperationalError` (БД недоступна) или `PoolTimeout` (нет соединений). +5. **Подтверждение в PostgreSQL**: `pg_stat_activity` показывает сессии с `wait_event_type=Lock` и держателя блокировки (`LOCK TABLE ...`). Действие: завершить транзакцию-держателя. Проверка: 5xx → 0, алерт resolved через 30–50 с после снятия блокировки (Docker 50,1 с, песочница 30,0 с: окно rate 1m заполняется успешными запросами, затем ближайшая оценка правила). + +### Инцидент 3: недоступна телеметрия (Tempo + Loki; затем сам Collector) + +Ожидаем: API не замечает отказа; Collector копит пакеты в ограниченной очереди, heap не выходит за `memory_limiter`; `TelemetryExportFailing` firing → resolved; после восстановления очередь досылается, потери посчитаны. Без Collector: SDK каждого процесса отбрасывает лишнее (очередь 2048), память API не растёт, потери видны в `otel_sdk_exporter_span_exported_total{error_type!=""}` после восстановления. + +Что показал эксперимент (числа — в таблицах выше): +- Tempo + Loki остановлены на 150 с под нагрузкой ~60 записей/с (Docker; в песочнице ~68/с): все ответы API 201 (9555 запросов), p95 не изменился (31 мс до и во время отказа). Очереди exporter Tempo/Loki росли (метрики продолжали идти в Prometheus — pipeline независимы), heap Collector вырос с 35,5 до max 86,9 MiB (песочница 44 → 93 MiB) — далеко от предела 200 MiB, память контейнера Collector не выше 107 MiB; `TelemetryExportFailing` перешёл в firing через 73 с (песочница ~100 с) — по заполнению очереди, ещё до первых потерь. После start очереди досланы за 60 с (песочница ~80 с), алерт resolved через 166 с после start (песочница ~155 с: окно rate 2m по счётчику потерь — последние пакеты с истёкшим окном повторов отброшены около момента восстановления, плюс `for`); все 13 пробных трасс, отправленных во время отказа, нашлись в Tempo (песочница 14 из 15). Потеряно и посчитано Collector-ом 4096 спанов и 235 записей логов (песочница 4096 и 269) — пакеты, у которых истекло окно повторов. +- Collector остановлен на 90 с: все ответы API 201 (5852 запроса), память двух реплик API 295 → max 314 MiB (+18 MiB; песочница +12 MiB) и после восстановления не растёт. Экспорт в SDK получает ошибку соединения, пакет отбрасывается (буфера на диске нет). В Docker остановленный контейнер исчезает из DNS сети, и каждая попытка экспорта висит до таймаута 5 с. Экспорт становится медленнее, чем API создаёт спаны, поэтому ограниченная очередь процессора (2048) переполняется и лишнее отбрасывается сразу: за отказ потеряно и посчитано 8252 спана `processor_queue_full` и 10752 `exporter_failed` (`otel_sdk_processor_span_processed_total`, `otel_sdk_exporter_span_exported_total` с `error_type`). В песочнице соединение отклонялось мгновенно: 24,6 тыс. `exporter_failed`, очередь не переполнялась. В обоих случаях память ограничена очередью и ответы API не замедлились (p95 31–32 мс); после start экспорт возобновился без перезапуска приложения (10287 спанов за минуту). +- Вывод: отказ любого звена телеметрии не затрагивает пользователей и не растит память без предела; потери ограничены временем отказа и видны в метриках. Короткую недоступность хранилища (до заполнения очереди Collector, здесь минуты) Collector переживает почти без потерь, недоступность самого Collector — нет. + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB. + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с | +| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) | + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 | +| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 | +| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 | +| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 | +| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | + ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается: недоступность любого хранилища телеметрии и самого Collector — API и обработка вложений продолжают работать; память ограничена очередями SDK (2048 на сигнал и процесс) и Collector (memory_limiter, очереди фиксированного размера). Переживают рестарт хранилищ (volumes): метрики, логи, трассы, настройки Grafana. +- Теряется (и считается): телеметрия, отправленная, пока Collector недоступен (SDK не буферизует на диск); пакеты, у которых истекло окно повторов Collector или очередь переполнена; буфер Collector при его рестарте. Метрики приложения за время недоступности Collector — пропуск на графиках (cumulative-счётчики после восстановления продолжают с верного итога). +- Один экземпляр каждого хранилища и Collector на одном Docker-host — нет HA телеметрии; при отказе Prometheus алерты не вычисляются (нужен внешний «мета-мониторинг» — вне учебного стенда). +- Метрики очереди считает один dispatcher: если он упал, значения `messenger_queue_*` устаревают (видно по отсутствию свежих точек); алерт `absent()` на них — следующий шаг. +- Учебные упрощения: внутренние соединения OTLP и UI без TLS (lab7), Prometheus/Loki/Tempo опубликованы на 127.0.0.1 для тестов и защиты; сэмплирование трасс 100% (для учебной нагрузки; стоимость и баланс сэмплирования — трек 5.1, не заявлен); burn-rate алерты с несколькими окнами — трек 5.2, не заявлен. +- Подтверждено измерением: алерты срабатывают и снимаются на контролируемых отказах; диагностика отличает остановленный worker, медленную БД и медленный обработчик; отказ телеметрии не влияет на API. Гипотеза без измерения: накладные расходы телеметрии на задержку и CPU (сравнение `OTEL_SDK_DISABLED=true/false` при одинаковой нагрузке) — трек 5.1. + +## Ответы на вопросы защиты (кратко) + +- **Лог, метрика и trace об одном инциденте.** Метрика отвечает «что и когда, насколько массово» (доля 5xx выросла в 14:02, только на POST messages) — дёшево, агрегатно, для алертов. Трасса — «где именно» в одном запросе (5 с из 5,004 с — `INSERT` в PostgreSQL), через все процессы. Лог — «почему» с деталями события (`QueryCanceled`, request_id, job_id). Связаны через `trace_id`: из алерта → error-трасса → её логи. +- **Почему `chat_id` в labels может исчерпать хранилище.** Каждая уникальная комбинация меток — отдельный временной ряд со своим индексом и чанками в памяти. Чатов — неограниченное и растущее число, умноженное на маршруты, коды и buckets гистограммы → миллионы рядов, рост RAM/диска TSDB и медленные запросы. Поэтому в метрики — только ограниченные множества, а ID — в трассы/логи. +- **Как очередь переносит причинную связь между процессами.** Producer записывает W3C `traceparent` (trace_id + id своего спана) в сообщение — здесь в строку outbox, затем dispatcher в AMQP header и поле тела; consumer извлекает его и начинает свой спан с этим родителем (или с link). Так trace_id один для HTTP-запроса и фоновой обработки, хотя между ними секунды и разные процессы. +- **Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик.** Среднее прячет хвост: 99 запросов по 10 мс и один по 5 с дают среднее 60 мс, а пользователь видит 5 с. Перцентиль — не аддитивная величина: p99 двух реплик с разной нагрузкой нельзя усреднить (реплика с 10 запросами весила бы как реплика с 10 000). Правильно — сложить buckets гистограмм всех реплик (`sum by (le)`) и уже из суммы вычислить `histogram_quantile`. diff --git a/attachments.py b/attachments.py new file mode 100644 index 0000000..dabbde2 --- /dev/null +++ b/attachments.py @@ -0,0 +1,80 @@ +"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202. + +Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого +объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение: + + 1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503. + 2. PUT исходника в staging/ — упало: удаляем объект (PUT мог пройти) + и строку, клиенту 503. Если и это не вышло — + uploading-строку уберёт janitor после grace. + 3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT + — упало/неизвестно: клиенту 503, строка + остаётся uploading -> janitor удалит объект + и строку. Если COMMIT всё же прошёл, задание + есть и будет обработано (повтор клиента даст + второе вложение — в контракте нет ключа). + 4. 202 Attachment(status=queued) — только после шага 3. + +Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный +брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно. +""" +import logging +from uuid import UUID, uuid4 + +import db +import settings +import storage +from errors import unavailable +from uploads import ImageInfo, Upload + +log = logging.getLogger("messenger.attachments") + + +def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict: + attachment_id = uuid4() + key = storage.new_key(uuid4().hex) + db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type, + upload.size, upload.sha256, image.width, image.height, key) + try: + storage.put(key, upload.file, upload.size, image.content_type, upload.sha256) + except storage.StorageUnavailable: + compensate(attachment_id, key) + raise unavailable("File storage is temporarily unavailable") from None + row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID) + if row is None: + # janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace). + log.warning("Upload %s was reclaimed before it was queued", attachment_id) + raise unavailable() + log.info("Attachment %s accepted into staging and queued", attachment_id) + return {**row, "variants": []} + + +def compensate(attachment_id: UUID, key: str) -> None: + """Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor.""" + try: + storage.delete(key, quick=True) + db.delete_upload_intent(attachment_id) + log.info("Upload %s rolled back after S3 failure", attachment_id) + except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту + log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__) + + +def public(row: dict) -> dict: + """Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей).""" + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["state"], + "variants": [ + {key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for v in row.get("variants", []) + ], + "error_code": row["error_code"] if row["state"] == "failed" else None, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..5f2a343 --- /dev/null +++ b/auth.py @@ -0,0 +1,150 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +import telemetry +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + # Argon2id нагружает CPU обработчика (а не БД): в трассе это отдельный спан. + with telemetry.internal_span("argon2id hash"): + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + with telemetry.internal_span("argon2id verify"): + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" +# Атрибуты спанов Redis: система и операция. Ключ (session:) и значения не пишутся. +REDIS_ATTRS = {"db.system.name": "redis", "server.address": "redis"} + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + with telemetry.client_span("redis SET", REDIS_ATTRS | {"db.operation.name": "SET"}): + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__, + extra={"event": "redis.unavailable", "error.type": type(exc).__name__}) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + with telemetry.client_span("redis GET", REDIS_ATTRS | {"db.operation.name": "GET"}): + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__, + extra={"event": "redis.unavailable", "error.type": type(exc).__name__}) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + with telemetry.client_span("redis DEL", REDIS_ATTRS | {"db.operation.name": "DEL"}): + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/broker.py b/broker.py new file mode 100644 index 0000000..9ec95f3 --- /dev/null +++ b/broker.py @@ -0,0 +1,104 @@ +"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms. + +Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует +scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure: +они только проверяют её пассивным declare и не могут создать/удалить очередь. + + exchange attachments (direct, durable) + --attachment.process.v1--> queue attachments.process.v1 (quorum, durable) + x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once), + x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря) + exchange attachments.dlx (direct, durable) + --attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py + через basic_get + nack не расходует delivery-limit) + +Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до +подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются. +""" +import json +import logging + +import pika +from pika.exceptions import AMQPError + +import pipeline +import settings + +log = logging.getLogger("messenger.broker") +logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал + +BrokerError = (AMQPError, OSError) + + +def parameters(url: str | None = None) -> pika.URLParameters: + params = pika.URLParameters(url or settings.BROKER_URL) + params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS + params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно + params.socket_timeout = 5 + params.connection_attempts = 1 + params.client_properties = {"connection_name": settings.INSTANCE_ID} + return params + + +def connect(url: str | None = None) -> pika.BlockingConnection: + return pika.BlockingConnection(parameters(url)) + + +def check_topology(channel) -> dict: + """Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях.""" + channel.exchange_declare(pipeline.EXCHANGE, passive=True) + main = channel.queue_declare(pipeline.QUEUE, passive=True).method + dlq = channel.queue_declare(pipeline.DLQ, passive=True).method + return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count} + + +def properties(message: dict) -> pika.BasicProperties: + # W3C Trace Context в native headers AMQP (основной carrier для worker) и в теле (поле схемы). + headers = {key: message[key] for key in ("traceparent", "tracestate") if key in message} + return pika.BasicProperties( + delivery_mode=pika.DeliveryMode.Persistent, + content_type="application/json", + message_id=f"{message['job_id']}:{message['attempt']}", + type=message["type"], + headers=headers or None, + ) + + +class Publisher: + """Канал в режиме confirm: basic_publish возвращает управление только после ack брокера. + + mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError. + Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях + строка outbox остаётся неопубликованной и будет отправлена позже. + """ + + def __init__(self): + self.connection = None + self.channel = None + + def open(self): + if self.channel is None or self.channel.is_closed or self.connection.is_closed: + self.close() + self.connection = connect() + self.channel = self.connection.channel() + self.channel.confirm_delivery() + check_topology(self.channel) + log.info("connected to broker, publisher confirms enabled", extra={"event": "broker.connected"}) + return self.channel + + def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY): + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True) + + def pump(self): + if self.connection is not None and self.connection.is_open: + self.connection.process_data_events(time_limit=0) # heartbeat в паузах + + def close(self): + try: + if self.connection is not None and self.connection.is_open: + self.connection.close() + except BrokerError: + pass + self.connection = None + self.channel = None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ecd322e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,494 @@ +name: messenger-lab6 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py. +x-pipeline-env: &pipeline-env + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + # Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults, + # доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0. + FAULT_INJECTION: ${FAULT_INJECTION:-1} + FAULTS_DIR: /faults + +# lab6: OpenTelemetry SDK всех процессов приложения -> OTel Collector (OTLP/HTTP, внутренняя сеть telemetry). +# Имена переменных — стандартные OTEL_* (их читает SDK); service.name задаётся в коде процесса и дублируется +# в OTEL_SERVICE_NAME у каждого сервиса. OTEL_SDK_DISABLED=true в .env выключает телеметрию целиком. +x-otel-env: &otel-env + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_TIMEOUT: "5" # секунды (Python SDK): экспорт не держит память дольше + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс + OTEL_BSP_MAX_QUEUE_SIZE: "2048" # спанов в памяти процесса, лишние отбрасываются и считаются + OTEL_BLRP_MAX_QUEUE_SIZE: "2048" # записей логов, так же + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" # otel.sdk.processor.*: очереди и потери SDK + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + +x-otel-depends: &otel-depends + # Порядок старта, но не жёсткая зависимость: без Collector приложение работает, телеметрия теряется. + condition: service_started + required: false + +x-alive-check: &alive-check + # Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад. + test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + +x-s3-api-secrets: &s3-api-secrets + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. + # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config + # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. + # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. + s3: + image: chrislusf/seaweedfs:3.80 + entrypoint: ["/usr/bin/weed"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=1024", "-volume.max=0", + "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: ["s3data:/data"] + networks: [data] + healthcheck: + # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») + test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 15s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. + s3-init: + build: *app-build + command: ["python", "s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL + OTEL_SERVICE_NAME: messenger-api + <<: [*s3-api-secrets, *otel-env] + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + redis: {condition: service_healthy} + otel-collector: *otel-depends + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. + janitor: + build: *app-build + command: ["python", "janitor.py"] + env_file: .env + environment: + CACHE_URL_FILE: "" # janitor не ходит в Redis + BROKER_URL_FILE: "" # и в брокер + OTEL_SERVICE_NAME: messenger-janitor + <<: [*s3-api-secrets, *otel-env] + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + otel-collector: *otel-depends + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + + # Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права — + # в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте. + # hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume. + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq + volumes: + - ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro + - rabbitmqdata:/var/lib/rabbitmq + secrets: [rabbitmq_definitions] + networks: [data] + healthcheck: + # Узел запущен, и vhost с очередями загружен (а не только «порт открыт»). + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"] + interval: 5s + timeout: 10s + retries: 24 + start_period: 20s + start_interval: 2s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает + # просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет. + dispatcher: + build: *app-build + command: ["python", "dispatcher.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_dispatcher + OTEL_SERVICE_NAME: messenger-dispatcher + <<: [*pipeline-env, *otel-env] + secrets: [database_url, broker_url_dispatcher] + volumes: ["faults:/faults"] + networks: [data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + otel-collector: *otel-depends + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + # Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем + # процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS. + # Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker. + worker: + build: *app-build + command: ["python", "worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_worker + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512} + OTEL_SERVICE_NAME: messenger-worker + <<: [*pipeline-env, *s3-api-secrets, *otel-env] + secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key] + volumes: ["faults:/faults"] + tmpfs: + - /tmp:size=256m,mode=1777 + networks: [data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + otel-collector: *otel-depends + healthcheck: *alive-check + restart: unless-stopped + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят; + # если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова. + stop_grace_period: 70s + deploy: + resources: + limits: {cpus: "2.00", memory: 1536M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # ---------------- lab6: наблюдаемость ---------------- + # OTel Collector: приём OTLP от приложения, memory_limiter + batch, ограниченные очереди exporter, + # отправка в Tempo / Loki / Prometheus. Не хранилище. Порты не публикуются (только сеть telemetry). + # Образ distroless (без shell), поэтому без healthcheck: здоровье видно по otelcol_* в Prometheus. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.164.0 + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro + networks: [telemetry] + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + # heap ограничивает memory_limiter (200 MiB); сверху — runtime Go и отображённый бинарник contrib. + limits: {cpus: "0.50", memory: 384M} + + # Одноразовый bootstrap прав на volumes Loki и Tempo (оба процесса работают как uid 10001, не root). + telemetry-init: + image: busybox:1.37.0 + user: root + command: ["sh", "-c", "chown -R 10001:10001 /loki /tempo"] + volumes: + - lokidata:/loki + - tempodata:/tempo + network_mode: none + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 32M} + + # Prometheus: метрики (OTLP receiver для Collector + pull служебных метрик), recording/alert rules. + # 127.0.0.1:9090 — только для evidence-скриптов и /alerts на защите. + prometheus: + image: prom/prometheus:v3.15.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --storage.tsdb.retention.size=1GB + - --web.enable-otlp-receiver + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - ./infra/prometheus/tests:/etc/prometheus/tests:ro # promtool test rules /etc/prometheus/tests/alerts_test.yml + - promdata:/prometheus + ports: + - "127.0.0.1:9090:9090" + networks: [telemetry, ops] + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Loki: логи (OTLP от Collector), файловое хранилище в volume lokidata. 127.0.0.1:3100 — для тестов. + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + ports: + - "127.0.0.1:3100:3100" + networks: [telemetry, ops] + depends_on: + telemetry-init: {condition: service_completed_successfully} + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Tempo: трассы (OTLP gRPC от Collector), блоки в volume tempodata. 127.0.0.1:3200 — для тестов. + tempo: + image: grafana/tempo:3.1.1 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + ports: + - "127.0.0.1:3200:3200" + networks: [telemetry, ops] + depends_on: + telemetry-init: {condition: service_completed_successfully} + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Grafana: интерфейс и запросы к трём хранилищам. Источники данных и dashboard — provisioning из Git + # (infra/grafana), ручного набора нет. Вход: admin / пароль из secrets/grafana_admin_password. + grafana: + image: grafana/grafana:12.4.14 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" # стенд не скачивает плагины из интернета при старте + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json + secrets: [grafana_admin_password] + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/etc/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + networks: [telemetry, ops] + depends_on: + prometheus: {condition: service_healthy} + loki: {condition: service_started} + tempo: {condition: service_started} + healthcheck: + # wget есть в образе на Alpine, curl — в варианте на Ubuntu: проверка работает в обоих. + test: ["CMD-SHELL", "wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health || curl -fsS -m 2 -o /dev/null http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 20s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 384M} + +networks: + edge: {} + data: + internal: true + # lab6: приложение -> Collector -> хранилища -> Grafana. Внутренняя: наружу ничего не публикуется. + telemetry: + internal: true + # Только для публикации UI/API хранилищ на 127.0.0.1 (internal-сеть не может публиковать порты). + ops: {} + +volumes: + pgdata: {} + redisdata: {} + s3data: {} + rabbitmqdata: {} + faults: {} + promdata: {} + lokidata: {} + tempodata: {} + grafanadata: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_config: {file: ./secrets/s3_config.json} + broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher} + broker_url_worker: {file: ./secrets/broker_url_worker} + rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json} + grafana_admin_password: {file: ./secrets/grafana_admin_password} diff --git a/db.py b/db.py new file mode 100644 index 0000000..96f57a9 --- /dev/null +++ b/db.py @@ -0,0 +1,452 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import pipeline +import settings +import telemetry +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +def check_connection(conn) -> None: + """Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса.""" + from opentelemetry.instrumentation.utils import suppress_instrumentation + with suppress_instrumentation(): + ConnectionPool.check_connection(conn) + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=check_connection, + open=False, +) + + +# lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»). +POOL_NAME = {"db.client.connection.pool.name": "messenger-api"} +POOL_WAIT = telemetry.meter.create_histogram( + "db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool") + + +def _pool_stats(): + try: + return pool.get_stats() + except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет + return None + + +def _observe_connections(_options): + from opentelemetry.metrics import Observation + stats = _pool_stats() + if not stats: + return [] + size, available = stats.get("pool_size", 0), stats.get("pool_available", 0) + return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}), + Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})] + + +def _observe_stat(key): + def callback(_options): + from opentelemetry.metrics import Observation + stats = _pool_stats() + return [Observation(stats.get(key, 0), POOL_NAME)] if stats else [] + return callback + + +telemetry.meter.create_observable_up_down_counter( + "db.client.connection.count", [_observe_connections], unit="{connection}", + description="Connections in the pool by state (used/idle)") +telemetry.meter.create_observable_up_down_counter( + "db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}", + description="Maximum pool size (DB_POOL_MAX)") +telemetry.meter.create_observable_up_down_counter( + "db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}", + description="Requests waiting for a free connection now") +telemetry.meter.create_observable_counter( + "db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}", + description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS") + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + started = time.perf_counter() + try: + with pool.connection(timeout=wait_seconds) as conn: + POOL_WAIT.record(time.perf_counter() - started, POOL_NAME) + yield conn + except PoolTimeout: + POOL_WAIT.record(time.perf_counter() - started, POOL_NAME) + raise + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__, + extra={"event": "db.unavailable", "error.type": type(exc).__name__}) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__, + extra={"event": "db.rejected_value", "error.type": type(exc).__name__}) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text", "filename") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" + + +def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() is not None + + +def check_member(chat_id: UUID, user_id: UUID) -> None: + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], + idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 + raise not_found("Chat not found") + if attachment_ids: + # Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409). + # FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние. + found = conn.execute( + """SELECT id, state FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""", + (attachment_ids, chat_id), + ).fetchall() + if len(found) != len(attachment_ids): + raise not_found("Attachment not found") + failed = any(row["state"] == "failed" for row in found) + else: + failed = False + saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m + WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""" + if failed: + # Повтор уже созданного сообщения возвращает прежний 201, даже если вложение + # с тех пор стало failed. Новое сообщение с failed-вложением — 409. + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if saved is None: + raise conflict("Failed attachment cannot be attached") + created = None + else: + created = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is None: + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if created is not None: + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES (%s, %s, %s, %s)""", + [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], + ) + return {**from_db(created), "attachment_ids": list(attachment_ids)} + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "m.chat_id = %s", [chat_id] + if after is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages m + WHERE {where} + ORDER BY m.created_at, m.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + +# ---------- вложения ---------- + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, " + "pipeline_version, created_at, updated_at, object_key") +VISIBLE_STATES = ("queued", "processing", "ready", "failed") + + +def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None: + """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging. + + Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт + и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД). + """ + try: + with connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, source_key, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", + (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, + width, height, source_key), + ) + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + + +def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None: + """Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией. + + После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL, + сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace). + """ + with connection() as conn: + with conn.transaction(): + row = conn.execute( + f"""UPDATE attachments SET state = 'queued', updated_at = now() + WHERE id = %s AND state = 'uploading' + RETURNING {ATTACHMENT_COLUMNS}""", + (attachment_id,), + ).fetchone() + if row is None: + return None + pipeline.enqueue_new(conn, attachment_id, actor) + return from_db(row) + + +def delete_upload_intent(attachment_id: UUID) -> None: + with connection(wait_seconds=1) as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) + + +def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: + """Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404. + + Возвращает строку и варианты текущей версии pipeline (они есть только у ready). + """ + with connection() as conn: + row = conn.execute( + f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} + FROM attachments a + JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s + WHERE a.id = %s AND a.state = ANY(%s)""", + (user_id, attachment_id, list(VISIBLE_STATES)), + ).fetchone() + if row is None: + raise not_found("Attachment not found") + variants = [] + if row["state"] == "ready": + variants = conn.execute( + """SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""", + (attachment_id, row["pipeline_version"]), + ).fetchall() + return {**from_db(row), "variants": variants} diff --git a/dispatcher.py b/dispatcher.py new file mode 100644 index 0000000..715d951 --- /dev/null +++ b/dispatcher.py @@ -0,0 +1,260 @@ +"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания. + +Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной +транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором +намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox +остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса, +ни ключей брокера, и недоступность брокера не мешает приёму файлов. + +Цикл: +1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP + LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT. + Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения, + который worker распознаёт по состоянию job (at-least-once, а не exactly-once). +2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed. +3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages). +Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру. +""" +import logging +import signal +import threading +import time +from pathlib import Path + +import psycopg +from opentelemetry.trace import SpanKind +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-dispatcher") + +import broker # noqa: E402 +import faults # noqa: E402 +import pipeline # noqa: E402 + +log = logging.getLogger("messenger.dispatcher") + +# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs, +# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому +# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик +# только читает последний снимок (не ходит в БД из потока экспорта). +STATS_SECONDS = 5 +STATS = {"at": 0.0, "db": None, "broker": None} +PUBLISHED = telemetry.meter.create_counter( + "messenger.messages.published", unit="{message}", + description="Outbox rows published to the broker (outcome: ok/error)") +QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE} + + +def _fresh(kind): + return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None + + +def _observe(kind, build): + def callback(_options): + from opentelemetry.metrics import Observation + data = _fresh(kind) + return [Observation(value, attrs) for value, attrs in build(data)] if data else [] + return callback + + +telemetry.meter.create_observable_gauge( + "messenger.queue.depth", [_observe("broker", lambda d: [ + (d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])], + unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)") +telemetry.meter.create_observable_gauge( + "messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])], + unit="{consumer}", description="Consumers subscribed to the queue") +telemetry.meter.create_observable_gauge( + "messenger.jobs.backlog", [_observe("db", lambda d: [ + (d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}), + (d["processing"], {"status": "processing"})])], + unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing") +telemetry.meter.create_observable_gauge( + "messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])], + unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits") +telemetry.meter.create_observable_gauge( + "messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])], + unit="{message}", description="Outbox rows due but not yet published") +telemetry.meter.create_observable_gauge( + "messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])], + unit="s", description="Age of the oldest due unpublished outbox row (publish lag)") + + +def collect_stats(conn, publisher) -> None: + """Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот.""" + if time.monotonic() - STATS["at"] < STATS_SECONDS: + return + db_stats = conn.execute( + """SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due, + count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed, + count(*) FILTER (WHERE status = 'processing') AS processing, + coalesce(extract(epoch FROM now() - min(available_at) + FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age + FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone() + outbox = conn.execute( + """SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due, + coalesce(extract(epoch FROM now() - min(available_at) + FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age + FROM outbox WHERE published_at IS NULL""").fetchone() + STATS["db"] = {**db_stats, **outbox} + try: + STATS["broker"] = broker.check_topology(publisher.open()) + except broker.BrokerError: + STATS["broker"] = None + publisher.close() + STATS["at"] = time.monotonic() + +STOP = threading.Event() +ACTOR = "dispatcher@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def publish_due(conn, publisher: broker.Publisher) -> int: + published = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, job_id, attempt, payload FROM outbox + WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", + (settings.DISPATCHER_BATCH,), + ).fetchall() + if not rows: + return published + for row in rows: + publish_one(conn, publisher, row) + published += 1 + log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)}) + if len(rows) < settings.DISPATCHER_BATCH: + return published + return published + + +def publish_one(conn, publisher: broker.Publisher, row: dict) -> None: + """Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст + уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу.""" + message = dict(row["payload"]) + job_id, attempt = message.get("job_id"), message.get("attempt") + attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send", + "messaging.destination.name": pipeline.EXCHANGE, + "messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY, + "messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id), + "messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]} + parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate")) + with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent, + kind=SpanKind.PRODUCER, attributes=attributes, + record_exception=False, set_status_on_exception=False) as span: + carrier = telemetry.inject() + if carrier.get("traceparent"): + message["traceparent"] = carrier["traceparent"] + message.pop("tracestate", None) + if carrier.get("tracestate"): + message["tracestate"] = carrier["tracestate"] + faults.crash("dispatcher_crash_before_publish") + try: + publisher.publish(message) + except broker.BrokerError as exc: + telemetry.mark_error(span, type(exc).__name__) + PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"}) + log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__, + extra={"event": "outbox.publish_failed", "job.id": str(job_id), + "error.type": type(exc).__name__}) + publisher.close() + raise + faults.crash("dispatcher_crash_after_confirm") + conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],)) + PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"}) + + +def note_failure(conn) -> None: + """Счётчик неудачных публикаций для наблюдения (ops.py status).""" + try: + conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1 + WHERE published_at IS NULL AND available_at <= now()""") + except psycopg.Error: + pass + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def run() -> None: + publisher = broker.Publisher() + conn = None + last_leases = last_reconcile = 0.0 + pause = 1.0 + while not STOP.is_set(): + try: + if conn is None or conn.closed: + conn = connect_db() + conn.execute("LISTEN outbox") + log.info("connected to PostgreSQL, listening for outbox notifications", + extra={"event": "db.connected"}) + touch_alive() + publish_due(conn, publisher) + collect_stats(conn, publisher) + now = time.monotonic() + if now - last_leases >= settings.DISPATCHER_POLL_SECONDS: + last_leases = now + if recovered := pipeline.recover_expired_leases(conn, ACTOR): + log.warning("recovered %s job(s) with expired lease", recovered, + extra={"event": "lease.recovered", "count": recovered}) + if now - last_reconcile >= 30: + last_reconcile = now + if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS): + log.warning("re-published %s queued job(s) without a message in flight", lost, + extra={"event": "outbox.republished", "count": lost}) + pause = 1.0 + publisher.pump() + # Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса. + for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1): + pass + except broker.BrokerError: + if conn is not None and not conn.closed: + note_failure(conn) + log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause, + extra={"event": "broker.unavailable"}) + STOP.wait(pause) + pause = min(pause * 2, 15) + except psycopg.Error as exc: + log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause, + extra={"event": "db.unavailable", "error.type": type(exc).__name__}) + try: + if conn is not None: + conn.close() + except psycopg.Error: + pass + conn = None + STOP.wait(pause) + pause = min(pause * 2, 15) + publisher.close() + if conn is not None: + conn.close() + log.info("stopped", extra={"event": "service.stopped"}) + telemetry.shutdown() + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID, + "ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"}) + run() + + +if __name__ == "__main__": + main() diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..cec62bc --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,123 @@ +# Runbooks алертов (lab6) + +Правила — `infra/prometheus/rules/alerts.yml`, unit-тесты правил — `infra/prometheus/tests/alerts_test.yml`. +Состояние алертов: http://127.0.0.1:9090/alerts или Grafana → Alerting → Alert rules (data source rules). +Dashboard: Grafana http://127.0.0.1:3000 → Messenger → «Messenger — lab6 observability». +Общий порядок: **метрика говорит «что и когда», трасса — «где именно», лог — «почему»**. + +Ссылки на трассу из лога: Grafana → Explore → Loki → запись → поле `trace_id` → «Открыть трассу». +Из спана в логи: Tempo → спан → «Logs for this span» (запрос по `trace_id`). + +--- + +## MessengerHighErrorRate + +**Что значит.** Больше 5% бизнес-запросов API (без `/health/*`) за минуту закончились 5xx, дольше минуты, +при трафике > 0,2 запроса/с. 4xx (неверный пароль, чужой чат) сюда не входят. + +**Влияние.** Пользователи получают 503/500, SLO-1 (99% без 5xx за час) расходует бюджет в 5+ раз быстрее нормы. + +**Найти причину.** +1. Какие маршруты и коды: dashboard «RPS по классу статуса» или + `sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))`. + 503 — недоступна зависимость, 500 — ошибка кода. +2. Медленная БД или медленный обработчик: + - `histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))` и + `db_client_connection_pending_requests` — запросы ждут соединение из пула → соединения заняты долгими запросами; + - Tempo, TraceQL `{ resource.service.name = "messenger-api" && status = error }` → открыть трассу: + если почти всё время server-спана занимает дочерний спан `INSERT`/`SELECT` (PostgreSQL) с ошибкой — + проблема в БД; если время в `argon2id hash`/`inspect image` или «пустое» время самого server-спана — в обработчике; + `redis GET` с ошибкой — Redis; `S3 PutObject` — SeaweedFS. +3. Логи API: `{service_name="messenger-api"} | event="db.unavailable"` (поле `error_type`: `QueryCanceled` — + statement_timeout 5 с, `PoolTimeout` — нет свободного соединения 3 с, `OperationalError` — БД недоступна); + `| event="redis.unavailable"`, `| event="s3.unavailable"`. +4. В PostgreSQL (если трассы указывают на БД): `docker compose exec postgres psql -U messenger -d messenger -c + "SELECT pid, state, wait_event_type, wait_event, now()-query_start AS age, left(query,60) FROM pg_stat_activity + WHERE datname='messenger' ORDER BY age DESC"` — блокировки (`wait_event_type = Lock`) и долгие запросы. + +**Исправить.** Снять блокировку/долгий запрос (`pg_terminate_backend(pid)`), восстановить зависимость +(`docker compose start postgres|redis|s3`). Не перезапускать API «на всякий случай»: причина не в нём. + +**Проверить восстановление.** Доля 5xx на dashboard → 0, алерт становится inactive через ~1–2 минуты +(окно rate 1m). Пример разбора: REPORT.md, инцидент 2. + +--- + +## MessengerQueueLagGrowing + +**Что значит.** Старейшее задание обработки вложения, которое уже можно брать (статус `queued`, +`available_at` прошёл), ждёт больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно. + +**Влияние.** Вложения остаются `queued`, клиенты не получают превью; SLO-3 (95% ready ≤ 30 с) нарушается. + +**Найти причину.** +1. `messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}` = 0 → worker не подключён + (см. MessengerWorkerNoConsumers). > 0 → worker есть, но не успевает или падает на заданиях. +2. Насыщение worker: `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` ≈ 1 и растёт + `messenger_queue_depth` → нужно больше потоков/реплик worker (`WORKER_CONCURRENCY`, `--scale worker=2`). +3. Ошибки обработки: `sum by (outcome) (rate(messenger_job_duration_seconds_count[5m]))` — много `retry`/`failed`; + логи `{service_name="messenger-worker"} | event=~"job.retry|job.failed"` (поле `error_type`). +4. Отличить от проблемы публикации: `messenger_outbox_oldest_age_seconds` большой → сообщения не доходят до + брокера (dispatcher/RabbitMQ), а не worker; лог `{service_name="messenger-dispatcher"} | event="broker.unavailable"`. +5. Трасса любой зависшей загрузки (по `trace_id` из лога API `event="http.request"` для POST attachments): + есть `publish attachments`, но нет `process attachments.process.v1` → сообщение в брокере, его никто не взял. + +**Исправить.** `docker compose start worker` (или `scripts/worker.sh start`); при насыщении — увеличить +параллелизм; при массовых ошибках — устранить их причину, затем `ops.py replay` для failed. + +**Проверить восстановление.** `messenger_jobs_backlog{status="queued"}` → 0, возраст → 0, вложения `ready`, +алерт inactive через ~15–30 с. Пример: REPORT.md, инцидент 1. + +--- + +## MessengerWorkerNoConsumers + +**Что значит.** RabbitMQ дольше минуты не видит ни одного подписчика очереди `attachments.process.v1`. + +**Найти причину.** `docker compose ps worker`; последние записи `{service_name="messenger-worker"}` в Loki +(`event="service.stopped"` — остановлен штатно; `event="broker.unavailable"` — не может подключиться к брокеру; +отсутствие записей и метрик `messenger_worker_capacity` — процесс не запущен). + +**Исправить.** Запустить worker / восстановить RabbitMQ. Принятые задания не теряются (outbox + durable очередь). + +--- + +## MessengerDbPoolSaturated + +**Что значит.** На реплике API запросы дольше минуты ждут свободное соединение из пула (`DB_POOL_MAX`=10). + +**Найти причину.** Почти всегда это следствие медленных запросов или блокировок в PostgreSQL, а не нехватки +соединений: те же шаги, что в MessengerHighErrorRate (трассы с долгими DB-спанами, `pg_stat_activity`). +Увеличивать пул имеет смысл, только если DB-спаны быстрые, а ожидание пула долгое при высоком RPS. + +--- + +## TelemetryExportFailing + +**Что значит.** Collector не может отправить данные в Tempo / Loki / Prometheus (метка `exporter`): очередь +exporter дольше минуты заполнена больше чем на 5% (хранилище не принимает), либо пакеты уже теряются — +у пакета истекло окно повторов 120 с (`send_failed`) или очередь переполнена (`enqueue_failed`). Окно повторов +считается для пакета с первой попытки, время ожидания в очереди в него не входит: очередь (200 пакетов) — +это и есть буфер на время недоступности. + +**Влияние.** На пользователей не влияет. Телеметрия этого периода частично потеряна (сколько — счётчики +`otelcol_exporter_send_failed_*`, `otelcol_exporter_enqueue_failed_*`). + +**Найти причину.** `docker compose ps tempo loki prometheus`; `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`; +логи Collector: `docker compose logs otel-collector --tail 50` (через Loki их не смотреть — это тот же конвейер). + +**Исправить.** `docker compose start tempo loki prometheus`. Очередь Collector досылается сама (в пределах окна +повторов). Проверить: `otelcol_exporter_queue_size` → 0, свежая трасса находится. Пример: REPORT.md, инцидент 3. + +--- + +## TelemetryCollectorDown + +**Что значит.** Prometheus не может забрать метрики Collector (`up{job="otel-collector"} == 0`). + +**Влияние.** Приложение работает; SDK каждого процесса держит не больше 2048 спанов и 2048 записей логов. +Потери считаются в самом процессе: пакет, который Collector не принял, — `otel_sdk_exporter_span_exported_total{error_type!=""}` +(обычно `ConnectionError`), переполнение очереди — `otel_sdk_processor_span_processed_total{error_type="queue_full"}`. Метрик приложения за +этот период нет (пропуск на графиках), cumulative-счётчики после восстановления продолжат расти без потери итогов. + +**Исправить.** `docker compose start otel-collector`; проверить, что трассы и метрики снова приходят. diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..a641476 --- /dev/null +++ b/errors.py @@ -0,0 +1,40 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..030c147 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-11-01 20:38:49, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_attachments.py +# exit code: 0, 10.3 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 23200a7c +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 0000000..6f5839a --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,18 @@ +# balancing: balancing, shared sessions/data/files, X-Instance-Id +# date: 2026-11-01 20:42:35, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 2.1 s + +20 x GET /users/me with one token -> replicas: {'1db2fedff78f': 10, '171a473b0d28': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 171a473b0d28; read back by ['171a473b0d28', '1db2fedff78f'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on 1db2fedff78f; old token then rejected by ['171a473b0d28', '1db2fedff78f'] +PASS: logout on one replica revokes the session on all of them +first POST on 1db2fedff78f -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae; same key on 171a473b0d28 -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +file uploaded via 1db2fedff78f; downloaded byte for byte via ['171a473b0d28', '1db2fedff78f'] +PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['171a473b0d28', '1db2fedff78f']; api containers: ['171a473b0d28', '1db2fedff78f'] +PASS: X-Instance-Id is in every response and points to a real container +6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..d9cad92 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-11-01 20:07:43, commit: 470d9d6 + uncommitted changes +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/configs.txt b/evidence/configs.txt new file mode 100644 index 0000000..add8075 --- /dev/null +++ b/evidence/configs.txt @@ -0,0 +1,32 @@ +# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning +# date: 2026-11-01 20:07:53, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/check_configs.py +# exit code: 0, 5.5 s + +--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0 + +--- promtool check config (infra/prometheus/prometheus.yml): exit 0 + +Checking /etc/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /etc/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool check rules (recording + alerts): exit 0 +Checking /etc/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /etc/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0 +SUCCESS +--- loki -verify-config (infra/loki/loki.yaml): exit 0 + +--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0 + +--- Grafana 12.4.14: database ok +datasource prometheus (prometheus): OK Successfully queried the Prometheus API. +datasource loki (loki): OK Data source successfully connected. +datasource tempo (tempo): OK Data source is working +dashboard 'Messenger — lab6 observability' in folder 'Messenger': 34 panels, provisioned=True, datasources ['loki', 'prometheus', 'tempo'] +PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana provisioned 3 healthy datasources and the dashboard without manual steps diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 0000000..f8ce011 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,47 @@ +# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage +# date: 2026-11-01 20:41:39, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_consistency.py +# exit code: 0, 159.9 s + +declared 9 000 000 bytes, sent 211817, closed the connection +declared 9 000 000 bytes, sent 3111817, closed the connection +rows in attachments: 63 -> 63; objects in bucket: 240 -> 240 +temporary upload files in api containers: {'171a473b0d28': 0, '1db2fedff78f': 0} (removed when the request ends) +api containers have no volumes: the original is never stored locally +PASS: an interrupted upload leaves no row, no object and no local file +crash after S3 PUT: row b1065ae1 state=uploading, object staging/b5763f64729a4b8a8c899bcd7009e599 exists in S3 +the half-written attachment is invisible: GET -> 404, attach to a message -> 404 +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.133} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.132} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.101} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.134} -> idempotent +the ready attachment of the same chat is untouched +PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period +object staging/23223961df3644dbbaa0fa4f40d57b82 put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.103} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.123} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 0, "seconds": 0.134} -> orphan deleted +ready attachments older than the grace period are untouched (they have rows) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=1 and keeps the row; download -> 503 (not 500) +PASS: orphans are found and deleted only after the grace period; missing objects are reported +--- S3 stopped (connection refused / name not resolved): docker compose stop s3 +upload -> 503 unavailable in 20234 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose start s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 259, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.142} +PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up +--- S3 frozen (no answer, timeouts): docker compose pause s3 +upload -> 503 unavailable in 14703 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose unpause s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 267, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.192} +PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up +5 passed, 0 failed diff --git a/evidence/dashboard.txt b/evidence/dashboard.txt new file mode 100644 index 0000000..e0daa60 --- /dev/null +++ b/evidence/dashboard.txt @@ -0,0 +1,36 @@ +# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels +# date: 2026-11-01 20:09:12, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/check_dashboard.py +# exit code: 0, 1.0 s + +PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident) + empty now: CPU контейнеров / лимит + empty now: Collector: потери и отказы + empty now: SDK приложений: экспорт и потери + empty now: alerts.yml: MessengerDbPoolSaturated + empty now: alerts.yml: MessengerHighErrorRate + empty now: alerts.yml: MessengerQueueLagGrowing + empty now: alerts.yml: MessengerWorkerNoConsumers + empty now: alerts.yml: TelemetryCollectorDown + empty now: alerts.yml: TelemetryExportFailing + empty now: Алерты Prometheus (pending/firing) + empty now: Память: RSS процесса и контейнер +LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error" +TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 4 trace(s) + +data: http_server_request_duration_seconds_count{job="messenger-api"} 48 series <- HTTP requests/duration (3) +data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3) +data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4) +data: messenger_queue_depth 2 series <- queue depth (4) +data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4) +data: db_client_connection_count 4 series <- DB pool used/idle (4) +data: db_client_connection_pending_requests 2 series <- DB pool waiting (4) +data: messenger_worker_busy 1 series <- worker busy (4) +data: messenger_worker_capacity 1 series <- worker capacity (4) +data: process_cpu_time_seconds_total 12 series <- process CPU (4) +data: process_memory_usage_bytes 6 series <- process RAM (4) +data: otelcol_exporter_queue_size 3 series <- collector queue (4) + +labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version +http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready'] +PASS: every dashboard/rule query runs, key signals have data, no forbidden labels diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 0000000..91eca92 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-11-01 20:42:53, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 11.6 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 71 -> 72 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 10; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 0000000..49f2722 --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,109 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 68448 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.007..0.019 rows=51 loops=1) + Buffers: shared hit=54 + -> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1) + Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Buffers: shared hit=54 + Planning: + Buffers: shared hit=8 + Planning Time: 0.067 ms + Execution Time: 0.026 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.007..0.012 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid))) + Buffers: shared hit=5 + Planning Time: 0.083 ms + Execution Time: 0.047 ms + + +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.024..0.176 rows=51 loops=1) + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1) + Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid))) + Buffers: shared hit=59 + SubPlan 1 + -> Sort (actual time=0.002..0.002 rows=0 loops=51) + Sort Key: a."position" + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Recheck Cond: (message_id = m.id) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Index Cond: (message_id = m.id) + Buffers: shared hit=51 + Planning: + Buffers: shared hit=63 + Planning Time: 0.210 ms + Execution Time: 0.193 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +-------------------------------------------------------------------------------- + Limit (actual time=4.087..4.090 rows=51 loops=1) + Buffers: shared hit=1255 + -> Sort (actual time=3.670..3.904 rows=10051 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 2377kB + Buffers: shared hit=1255 + -> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1) + Filter: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Rows Removed by Filter: 53448 + Buffers: shared hit=1255 + Planning Time: 0.041 ms + Execution Time: 4.136 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.375..1.378 rows=51 loops=1) + Buffers: shared hit=1006 + -> Sort (actual time=1.374..1.376 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=1006 + -> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1) + Recheck Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Filter: (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid)) + Rows Removed by Filter: 10000 + Heap Blocks: exact=991 + Buffers: shared hit=1006 + -> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1) + Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Buffers: shared hit=15 + Planning: + Buffers: shared hit=4 + Planning Time: 0.057 ms + Execution Time: 1.395 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..1824deb --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,46 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-11-01 20:42:57, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 1.2 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=54 + -> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1) + Buffers: shared hit=54 + Buffers: shared hit=8 + Execution Time: 0.026 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Buffers: shared hit=5 + Execution Time: 0.047 ms +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1) + Buffers: shared hit=59 + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + Buffers: shared hit=63 + Execution Time: 0.193 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=1255 + Buffers: shared hit=1255 + -> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1) + Buffers: shared hit=1255 + Execution Time: 4.136 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=1006 + Buffers: shared hit=1006 + -> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1) + Buffers: shared hit=1006 + -> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1) + Buffers: shared hit=15 + Buffers: shared hit=4 + Execution Time: 1.395 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..8fb5f43 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica, file downloaded via the other one (make test-failover) +# date: 2026-11-01 20:43:39, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 42.0 s + +hook: scripts\stop-one.cmd; api replicas: ['171a473b0d28', '1db2fedff78f'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Up 41 seconds (healthy) +api 1db2fedff78f Up 41 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.75} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Exited (0) 11 seconds ago +api 1db2fedff78f Up 58 seconds (healthy) +the hook stopped 171a473b0d28 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 171a473b0d28 -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Up 2 seconds (healthy) +api 1db2fedff78f Up About a minute (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.578} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Up 19 seconds (healthy) +api 1db2fedff78f Exited (0) 11 seconds ago +the hook stopped 1db2fedff78f (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 1db2fedff78f -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 0000000..3f08921 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-11-01 20:45:06, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 55.6 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (8016 ms) +valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1016 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..63cdb70 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-11-01 20:42:37, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.2 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'171a473b0d28': 10, '1db2fedff78f': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/incident-collector-down.json b/evidence/incident-collector-down.json new file mode 100644 index 0000000..c3d183a --- /dev/null +++ b/evidence/incident-collector-down.json @@ -0,0 +1,39 @@ +{ + "scenario": "collector-down", + "date": "2026-11-01 20:25:18", + "outage_s": 90, + "baseline": { + "requests": 870, + "codes": { + "201": 870 + }, + "success_ratio": 1.0, + "success_rps": 58.0, + "p95_ms_successful": 32.0 + }, + "during": { + "requests": 5852, + "codes": { + "201": 5852 + }, + "success_ratio": 1.0, + "success_rps": 57.9, + "p95_ms_successful": 31.0 + }, + "api_mib_before": 295.2, + "api_mib_max": 313.5, + "sdk_lost_spans_api": { + "processor_queue_full": 8252, + "exporter_failed": 10752 + }, + "sdk_exported_ok_after_restart": 10287, + "after": { + "requests": 1403, + "codes": { + "201": 1403 + }, + "success_ratio": 1.0, + "success_rps": 56.1, + "p95_ms_successful": 32.0 + } +} \ No newline at end of file diff --git a/evidence/incident-collector.txt b/evidence/incident-collector.txt new file mode 100644 index 0000000..87b1dbb --- /dev/null +++ b/evidence/incident-collector.txt @@ -0,0 +1,20 @@ +# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted +# date: 2026-11-01 20:27:53, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py collector-down +# exit code: 0, 155.8 s + +=== collector-down +[ 24.6s] baseline: {'requests': 870, 'codes': {'201': 870}, 'success_ratio': 1.0, 'success_rps': 58.0, 'p95_ms_successful': 32.0}, api containers 295.2 MiB +[ 25.3s] docker compose stop otel-collector +[ 29.3s] {'t': 0.7, 'api_mib': 292.8} +[ 43.6s] {'t': 14.7, 'api_mib': 298.9} +[ 58.0s] {'t': 29.0, 'api_mib': 306.5} +[ 72.6s] {'t': 43.4, 'api_mib': 307.8} +[ 87.0s] {'t': 58.0, 'api_mib': 308.2} +[ 101.0s] {'t': 72.4, 'api_mib': 313.5} +[ 115.0s] {'t': 86.4, 'api_mib': 313.3} +[ 125.6s] docker compose start otel-collector +[ 155.6s] API during collector outage: {'requests': 5852, 'codes': {'201': 5852}, 'success_ratio': 1.0, 'success_rps': 57.9, 'p95_ms_successful': 31.0}; api memory 295.2 -> max 313.5 MiB; SDK lost spans (api): {'processor_queue_full': 8252, 'exporter_failed': 10752}; exported OK in the last minute after restart: 10287 +[ 155.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-collector-down.json +PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start +1 passed, 0 failed diff --git a/evidence/incident-db-slow.json b/evidence/incident-db-slow.json new file mode 100644 index 0000000..2bc2919 --- /dev/null +++ b/evidence/incident-db-slow.json @@ -0,0 +1,82 @@ +{ + "scenario": "db-slow", + "date": "2026-11-01 20:13:54", + "lock_seconds": 210, + "baseline": { + "requests": 2310, + "codes": { + "201": 2310 + }, + "success_ratio": 1.0, + "success_rps": 115.5, + "p95_ms_successful": 32.0, + "server_p95_s": 0.019 + }, + "firing_after_lock_s": 140.8, + "diagnosis": { + "http_5xx_ratio_1m": 1.0, + "http_p95_messages_s": 7.375, + "pool_used": 8.0, + "pool_wait_p95_s": 0.001, + "status_codes_1m": { + "201": 0.0, + "200": 0.0, + "404": 0.0, + "400": 0.0, + "409": 0.0, + "503": 1.6 + }, + "reads_during_lock_ms": { + "GET /users/me + GET /chats, avg per request": 17.2, + "max": 23.5 + }, + "api_log_db_unavailable_by_error_type": { + "QueryCanceled": 216 + }, + "error_trace": { + "trace_id": "237e2d6b1c7db22adbb7d4752fa47f2", + "server_ms": 5013.3, + "slowest_db_span": "INSERT", + "slowest_db_ms": 5001.3, + "db_span_status": "ERROR", + "db_share": 0.998, + "tree": [ + "- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503", + " - [api] redis GET (CLIENT, 1.1 ms)", + " - [api] SELECT (CLIENT, 1.6 ms)", + " - [api] INSERT (CLIENT, 5001.3 ms) ERROR" + ] + }, + "handler_cpu_contrast": { + "trace_id": "4a222e478c2d51006538a7bfeb485eb1", + "server_ms": 21.2, + "argon2id_ms": 16.4, + "db_ms_total": 0.7, + "tree": [ + "- [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201", + " - [api] argon2id hash (INTERNAL, 16.4 ms)", + " - [api] INSERT (CLIENT, 0.7 ms)" + ] + } + }, + "during_lock": { + "requests": 328, + "codes": { + "503": 320, + "201": 8 + }, + "success_ratio": 0.0244, + "success_rps": 0.0, + "p95_ms_successful": 2156.0 + }, + "resolved_after_unlock_s": 50.1, + "after": { + "requests": 5269, + "codes": { + "201": 5269 + }, + "success_ratio": 1.0, + "success_rps": 116.8, + "p95_ms_successful": 31.0 + } +} \ No newline at end of file diff --git a/evidence/incident-db.txt b/evidence/incident-db.txt new file mode 100644 index 0000000..a5e8378 --- /dev/null +++ b/evidence/incident-db.txt @@ -0,0 +1,64 @@ +# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved +# date: 2026-11-01 20:18:39, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py db-slow +# exit code: 0, 285.7 s + +=== db-slow +[ 0.0s] load: 8 threads POST /messages +[ 25.0s] baseline: {'requests': 2310, 'codes': {'201': 2310}, 'success_ratio': 1.0, 'success_rps': 115.5, 'p95_ms_successful': 32.0, 'server_p95_s': 0.019} +[ 25.0s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait) +[ 25.1s] 5xx ratio=None p95=0.019 s pool used=0.0 waiting=0.0 +[ 30.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0 +[ 35.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0 +[ 40.1s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0 +[ 45.2s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0 +[ 50.2s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0 +[ 55.3s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0 +[ 60.3s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 65.3s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0 +[ 70.4s] 5xx ratio=0.0 p95=0.045 s pool used=8.0 waiting=0.0 +[ 75.4s] 5xx ratio=0.015 p95=7.18 s pool used=8.0 waiting=0.0 +[ 80.4s] 5xx ratio=0.015 p95=7.234 s pool used=8.0 waiting=0.0 +[ 85.4s] 5xx ratio=0.015 p95=7.375 s pool used=8.0 waiting=0.0 +[ 90.4s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0 +[ 95.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0 +[ 100.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0 +[ 105.5s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 110.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 115.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 120.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 125.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 130.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 135.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 140.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 145.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 150.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 155.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 160.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 165.8s] ALERT MessengerHighErrorRate firing (140.8 s after lock) +[ 172.2s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 17.2, "max": 23.5}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 216}} +[ 172.2s] error trace (DB): {"trace_id": "237e2d6b1c7db22adbb7d4752fa47f2", "server_ms": 5013.3, "slowest_db_span": "INSERT", "slowest_db_ms": 5001.3, "db_span_status": "ERROR", "db_share": 0.998} + - [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503 + - [api] redis GET (CLIENT, 1.1 ms) + - [api] SELECT (CLIENT, 1.6 ms) + - [api] INSERT (CLIENT, 5001.3 ms) ERROR +[ 172.2s] contrast, slow handler (CPU): {"trace_id": "4a222e478c2d51006538a7bfeb485eb1", "server_ms": 21.2, "argon2id_ms": 16.4, "db_ms_total": 0.7} + - [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201 + - [api] argon2id hash (INTERNAL, 16.4 ms) + - [api] INSERT (CLIENT, 0.7 ms) +[ 235.5s] lock released (psql exit 0), 210.5 s after LOCK +[ 235.5s] ... MessengerHighErrorRate: firing (0 s) +[ 240.5s] ... MessengerHighErrorRate: firing (5 s) +[ 245.5s] ... MessengerHighErrorRate: firing (10 s) +[ 250.5s] ... MessengerHighErrorRate: firing (15 s) +[ 255.5s] ... MessengerHighErrorRate: firing (20 s) +[ 260.6s] ... MessengerHighErrorRate: firing (25 s) +[ 265.6s] ... MessengerHighErrorRate: firing (30 s) +[ 270.6s] ... MessengerHighErrorRate: firing (35 s) +[ 275.6s] ... MessengerHighErrorRate: firing (40 s) +[ 280.6s] ... MessengerHighErrorRate: firing (45 s) +[ 285.6s] ALERT MessengerHighErrorRate resolved 50.1 s after unlock +[ 285.6s] load: during lock {'requests': 328, 'codes': {'503': 320, '201': 8}, 'success_ratio': 0.0244, 'success_rps': 0.0, 'p95_ms_successful': 2156.0}; after {'requests': 5269, 'codes': {'201': 5269}, 'success_ratio': 1.0, 'success_rps': 116.8, 'p95_ms_successful': 31.0} +[ 285.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-db-slow.json +PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved +1 passed, 0 failed diff --git a/evidence/incident-telemetry-down.json b/evidence/incident-telemetry-down.json new file mode 100644 index 0000000..40321f2 --- /dev/null +++ b/evidence/incident-telemetry-down.json @@ -0,0 +1,273 @@ +{ + "scenario": "telemetry-down", + "date": "2026-11-01 20:18:47", + "stopped": [ + "tempo", + "loki" + ], + "outage_s": 150, + "baseline": { + "requests": 899, + "codes": { + "201": 899 + }, + "success_ratio": 1.0, + "success_rps": 59.9, + "p95_ms_successful": 31.0 + }, + "collector_heap_before_mib": 35.5, + "collector_container_before_mib": 75.9, + "during": { + "requests": 9555, + "codes": { + "201": 9555 + }, + "success_ratio": 1.0, + "success_rps": 59.6, + "p95_ms_successful": 31.0 + }, + "samples": [ + { + "t": 0.0, + "collector_heap_mib": 56.4, + "collector_rss_mib": 237.6, + "collector_container_mib": 76.0, + "queue": { + "otlp_http/loki": 12.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 5.0 + }, + "alert": "inactive" + }, + { + "t": 12.1, + "collector_heap_mib": 45.4, + "collector_rss_mib": 237.8, + "collector_container_mib": 76.0, + "queue": { + "otlp_http/loki": 20.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 9.0 + }, + "alert": "pending" + }, + { + "t": 24.2, + "collector_heap_mib": 54.2, + "collector_rss_mib": 237.8, + "collector_container_mib": 76.0, + "queue": { + "otlp_http/loki": 27.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 12.0 + }, + "alert": "pending" + }, + { + "t": 36.2, + "collector_heap_mib": 54.2, + "collector_rss_mib": 237.8, + "collector_container_mib": 78.1, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 15.0 + }, + "alert": "pending" + }, + { + "t": 48.7, + "collector_heap_mib": 62.9, + "collector_rss_mib": 240.4, + "collector_container_mib": 82.2, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 15.0 + }, + "alert": "pending" + }, + { + "t": 60.8, + "collector_heap_mib": 51.2, + "collector_rss_mib": 244.4, + "collector_container_mib": 82.2, + "queue": { + "otlp_http/loki": 42.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 19.0 + }, + "alert": "pending" + }, + { + "t": 73.1, + "collector_heap_mib": 59.9, + "collector_rss_mib": 244.4, + "collector_container_mib": 82.3, + "queue": { + "otlp_http/loki": 50.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 22.0 + }, + "alert": "firing" + }, + { + "t": 85.2, + "collector_heap_mib": 70.5, + "collector_rss_mib": 248.4, + "collector_container_mib": 88.7, + "queue": { + "otlp_http/loki": 57.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 26.0 + }, + "alert": "firing" + }, + { + "t": 97.7, + "collector_heap_mib": 70.5, + "collector_rss_mib": 248.4, + "collector_container_mib": 94.5, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 27.0 + }, + "alert": "firing" + }, + { + "t": 109.8, + "collector_heap_mib": 56.3, + "collector_rss_mib": 256.7, + "collector_container_mib": 94.5, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 27.0 + }, + "alert": "firing" + }, + { + "t": 122.1, + "collector_heap_mib": 67.5, + "collector_rss_mib": 256.7, + "collector_container_mib": 94.5, + "queue": { + "otlp_http/loki": 70.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 31.0 + }, + "alert": "firing" + }, + { + "t": 134.2, + "collector_heap_mib": 76.1, + "collector_rss_mib": 256.9, + "collector_container_mib": 98.8, + "queue": { + "otlp_http/loki": 78.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 34.0 + }, + "alert": "firing" + }, + { + "t": 146.2, + "collector_heap_mib": 86.9, + "collector_rss_mib": 267.1, + "collector_container_mib": 107.0, + "queue": { + "otlp_http/loki": 85.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 38.0 + }, + "alert": "firing" + } + ], + "collector_heap_max_mib": 86.9, + "collector_rss_max_mib": 267.1, + "collector_container_max_mib": 107.0, + "TelemetryExportFailing_firing_at_s": 73.1, + "queue_drained_after_start_s": 60.1, + "probe_traces": [ + { + "sent_at_s": 0.0, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 12.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 24.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 36.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 48.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 60.8, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 73.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 85.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 97.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 109.8, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 122.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 134.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 146.2, + "http": 201, + "trace_in_tempo": true + } + ], + "lost_in_collector": { + "spans": 4096, + "log_records": 235 + }, + "resolved_after_start_s": 165.6, + "after": { + "requests": 9370, + "codes": { + "201": 9370 + }, + "success_ratio": 1.0, + "success_rps": 58.3, + "p95_ms_successful": 31.0 + } +} \ No newline at end of file diff --git a/evidence/incident-telemetry.txt b/evidence/incident-telemetry.txt new file mode 100644 index 0000000..32e937c --- /dev/null +++ b/evidence/incident-telemetry.txt @@ -0,0 +1,47 @@ +# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted +# date: 2026-11-01 20:25:06, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py telemetry-down +# exit code: 0, 379.8 s + +=== telemetry-down +[ 22.5s] baseline: {'requests': 899, 'codes': {'201': 899}, 'success_ratio': 1.0, 'success_rps': 59.9, 'p95_ms_successful': 31.0}, collector heap 35.5 MiB, container 75.9 MiB +[ 53.8s] docker compose stop tempo loki: stopped +[ 55.9s] {'t': 0.0, 'collector_heap_mib': 56.4, 'collector_rss_mib': 237.6, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 5.0}, 'alert': 'inactive'} +[ 68.0s] {'t': 12.1, 'collector_heap_mib': 45.4, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 9.0}, 'alert': 'pending'} +[ 80.0s] {'t': 24.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 12.0}, 'alert': 'pending'} +[ 92.5s] {'t': 36.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 78.1, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'} +[ 104.5s] {'t': 48.7, 'collector_heap_mib': 62.9, 'collector_rss_mib': 240.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'} +[ 116.9s] {'t': 60.8, 'collector_heap_mib': 51.2, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'} +[ 128.9s] {'t': 73.1, 'collector_heap_mib': 59.9, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.3, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'firing'} +[ 141.5s] {'t': 85.2, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 88.7, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'firing'} +[ 153.5s] {'t': 97.7, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'} +[ 165.9s] {'t': 109.8, 'collector_heap_mib': 56.3, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'} +[ 178.0s] {'t': 122.1, 'collector_heap_mib': 67.5, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 70.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 31.0}, 'alert': 'firing'} +[ 190.0s] {'t': 134.2, 'collector_heap_mib': 76.1, 'collector_rss_mib': 256.9, 'collector_container_mib': 98.8, 'queue': {'otlp_http/loki': 78.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 34.0}, 'alert': 'firing'} +[ 202.6s] {'t': 146.2, 'collector_heap_mib': 86.9, 'collector_rss_mib': 267.1, 'collector_container_mib': 107.0, 'queue': {'otlp_http/loki': 85.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 38.0}, 'alert': 'firing'} +[ 214.0s] docker compose start tempo loki +[ 274.1s] exporter queues drained 60.1 s after start +[ 289.4s] ... TelemetryExportFailing: firing (0 s) +[ 294.4s] ... TelemetryExportFailing: firing (5 s) +[ 299.5s] ... TelemetryExportFailing: firing (10 s) +[ 304.5s] ... TelemetryExportFailing: firing (15 s) +[ 309.5s] ... TelemetryExportFailing: firing (20 s) +[ 314.5s] ... TelemetryExportFailing: firing (25 s) +[ 319.5s] ... TelemetryExportFailing: firing (30 s) +[ 324.5s] ... TelemetryExportFailing: firing (35 s) +[ 329.5s] ... TelemetryExportFailing: firing (40 s) +[ 334.5s] ... TelemetryExportFailing: firing (45 s) +[ 339.6s] ... TelemetryExportFailing: firing (50 s) +[ 344.6s] ... TelemetryExportFailing: firing (55 s) +[ 349.6s] ... TelemetryExportFailing: firing (60 s) +[ 354.6s] ... TelemetryExportFailing: firing (65 s) +[ 359.6s] ... TelemetryExportFailing: firing (70 s) +[ 364.6s] ... TelemetryExportFailing: firing (75 s) +[ 369.6s] ... TelemetryExportFailing: firing (80 s) +[ 374.7s] ... TelemetryExportFailing: firing (85 s) +[ 379.7s] API during outage: {'requests': 9555, 'codes': {'201': 9555}, 'success_ratio': 1.0, 'success_rps': 59.6, 'p95_ms_successful': 31.0} +[ 379.7s] probe traces: 13/13 reached Tempo (lost: sent at [] s) +[ 379.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 235}; collector heap 35.5 -> max 86.9 MiB (memory_limiter 200 MiB), container max 107.0 MiB +[ 379.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-telemetry-down.json +PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved +1 passed, 0 failed diff --git a/evidence/incident-worker-stop.json b/evidence/incident-worker-stop.json new file mode 100644 index 0000000..d1b90eb --- /dev/null +++ b/evidence/incident-worker-stop.json @@ -0,0 +1,50 @@ +{ + "scenario": "worker-stop", + "date": "2026-11-01 20:09:59", + "firing_after_stop_s": { + "MessengerWorkerNoConsumers": 101.7, + "MessengerQueueLagGrowing": 131.8 + }, + "diagnosis": { + "queue_consumers": 0.0, + "broker_queue_depth": 6.0, + "jobs_backlog_queued": 3.0, + "oldest_job_age_s": 127.3, + "outbox_oldest_age_s": 0.0, + "published_ok_last_5m": 16.41239671427261, + "worker_metrics_last_seen_s": 131.0, + "last_worker_log": { + "event": "service.stopped", + "message": "stopped" + }, + "upload_trace_span_names": [ + "INSERT", + "NOTIFY", + "POST /api/v1/chats/{chat_id}/attachments", + "S3 PutObject", + "SELECT", + "UPDATE", + "create attachments", + "inspect image", + "publish attachments", + "redis GET", + "republish lost message" + ], + "upload_trace_has_consumer_span": false + }, + "ready_after_start_s": [ + 4.2, + 4.2, + 4.7 + ], + "resolved_after_start_s": { + "MessengerWorkerNoConsumers": 29.7, + "MessengerQueueLagGrowing": 29.7 + }, + "after_recovery": { + "jobs_backlog_queued": 0.0, + "oldest_job_age_s": 0.0, + "queue_consumers": 2.0 + }, + "time_to_ready_p95_5m_s": 282.0 +} \ No newline at end of file diff --git a/evidence/incident-worker.txt b/evidence/incident-worker.txt new file mode 100644 index 0000000..b156c4a --- /dev/null +++ b/evidence/incident-worker.txt @@ -0,0 +1,49 @@ +# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved +# date: 2026-11-01 20:12:40, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py worker-stop +# exit code: 0, 161.8 s + +=== worker-stop +[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s +[ 1.2s] docker compose stop worker +[ 1.3s] uploaded 3 attachments: 202 queued (API accepts without worker) +[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {} +[ 6.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {} +[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {} +[ 16.5s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {} +[ 21.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {} +[ 26.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {} +[ 31.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {} +[ 36.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {} +[ 41.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 46.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 51.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 56.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 61.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 66.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 71.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 76.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 81.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 86.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 91.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 96.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 101.8s] ALERT MessengerWorkerNoConsumers firing (101.7 s after stop) +[ 101.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 106.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 111.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 116.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 121.8s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 126.9s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 131.9s] ALERT MessengerQueueLagGrowing firing (131.8 s after stop) +[ 132.0s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 6.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 127.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 16.41239671427261, "worker_metrics_last_seen_s": 131.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET", "republish lost message"], "upload_trace_has_consumer_span": false} +[ 135.3s] docker compose start worker +[ 136.6s] all attachments ready 4.7 s after worker start; backlog now 3.0 +[ 136.6s] ... MessengerWorkerNoConsumers: firing (0 s) +[ 141.6s] ... MessengerWorkerNoConsumers: firing (5 s) +[ 146.6s] ... MessengerWorkerNoConsumers: firing (10 s) +[ 151.7s] ... MessengerWorkerNoConsumers: firing (15 s) +[ 156.7s] ... MessengerWorkerNoConsumers: firing (20 s) +[ 161.7s] alerts resolved: {'MessengerWorkerNoConsumers': 29.7, 'MessengerQueueLagGrowing': 29.7}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0} +[ 161.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-worker-stop.json +PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved +1 passed, 0 failed diff --git a/evidence/jobs-concurrency.txt b/evidence/jobs-concurrency.txt new file mode 100644 index 0000000..87f2616 --- /dev/null +++ b/evidence/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-11-01 20:38:38, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py concurrency +# exit code: 0, 15.8 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) +1 passed, 0 failed diff --git a/evidence/jobs-crash.txt b/evidence/jobs-crash.txt new file mode 100644 index 0000000..eade149 --- /dev/null +++ b/evidence/jobs-crash.txt @@ -0,0 +1,15 @@ +# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease +# date: 2026-11-01 20:33:58, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease +# exit code: 0, 157.0 s + +--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат. +worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] +--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack. +worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) +--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit). +staging source still present after the crash (worker died before cleanup); waiting 60 s window +janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone +--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; +frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] +4 passed, 0 failed diff --git a/evidence/jobs-janitor-replay.txt b/evidence/jobs-janitor-replay.txt new file mode 100644 index 0000000..800d25f --- /dev/null +++ b/evidence/jobs-janitor-replay.txt @@ -0,0 +1,13 @@ +# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix) +# date: 2026-11-01 20:38:23, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_janitor_replay.py +# exit code: 0, 187.0 s + +race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +waiting 86 s until the leftovers are older than grace (120 s) +janitor pass over the bucket: did not try to delete any object of the replayable failed attachment +ops replay -> job 09f5cf6c-bca3-4759-9930-8d18968ed1be ready +after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 0, deleted 0 +retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)" +1 passed, 0 failed diff --git a/evidence/jobs-limits.txt b/evidence/jobs-limits.txt new file mode 100644 index 0000000..8fd987d --- /dev/null +++ b/evidence/jobs-limits.txt @@ -0,0 +1,11 @@ +# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) +# date: 2026-11-01 20:30:08, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py limits +# exit code: 0, 72.7 s + +--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker. +10 MiB + 8 bytes -> 413 before any job is created +imaging.py on 8000x5000 PNG: exit 0, peak RSS 226.9 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large) +memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived +imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s +1 passed, 0 failed diff --git a/evidence/jobs-outbox.txt b/evidence/jobs-outbox.txt new file mode 100644 index 0000000..15d44c4 --- /dev/null +++ b/evidence/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-11-01 20:31:21, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py outbox-crash +# exit code: 0, 17.2 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/jobs-recovery.txt b/evidence/jobs-recovery.txt new file mode 100644 index 0000000..ff53686 --- /dev/null +++ b/evidence/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-11-01 20:31:04, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 55.2 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment 0d7622f1-7c85-4c52-9dc0-2dad8f729c8d +message with attachment 201, content 409, other messages 201; job 6a45774d-0d88-46ec-87b9-70fb4c122bbd published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job 34eb3c30-1942-4777-bce2-1ea2a4eca4c9 +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/jobs-retry.txt b/evidence/jobs-retry.txt new file mode 100644 index 0000000..7dcf40b --- /dev/null +++ b/evidence/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-11-01 20:35:16, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 77.6 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] +4 passed, 0 failed diff --git a/evidence/jobs-variants.txt b/evidence/jobs-variants.txt new file mode 100644 index 0000000..7a843a4 --- /dev/null +++ b/evidence/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-11-01 20:28:56, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py variants +# exit code: 0, 20.1 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..ae3d0d2 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 4, + "successful_requests": 7178, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 239.02, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 47.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..01637d0 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-11-01 20:44:10, commit: 470d9d6 + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.3 s + +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 4, + "successful_requests": 7178, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 239.02, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 47.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/attachments.txt b/evidence/native-sandbox/attachments.txt new file mode 100644 index 0000000..516518a --- /dev/null +++ b/evidence/native-sandbox/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-11-01 00:52:47, commit: unknown +# command: python student-tests/test_attachments.py +# exit code: 0, 13.5 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f30c028f +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (70 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (84 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/native-sandbox/check.txt b/evidence/native-sandbox/check.txt new file mode 100644 index 0000000..6bb54d6 --- /dev/null +++ b/evidence/native-sandbox/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-11-01 00:22:29, commit: unknown +# command: python scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/native-sandbox/configs.txt b/evidence/native-sandbox/configs.txt new file mode 100644 index 0000000..575837b --- /dev/null +++ b/evidence/native-sandbox/configs.txt @@ -0,0 +1,28 @@ +# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning +# date: 2026-11-01 00:22:31, commit: unknown +# command: python student-tests/check_configs.py +# exit code: 0, 0.7 s + +--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0 + +--- promtool check config (infra/prometheus/prometheus.yml): exit 0 + +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool check rules (recording + alerts): exit 0 +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0 +SUCCESS +--- loki -verify-config (infra/loki/loki.yaml): exit 0 + +--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0 + +--- Grafana: not reachable at http://127.0.0.1:3000, skipped (local binaries mode) +PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid diff --git a/evidence/native-sandbox/dashboard.txt b/evidence/native-sandbox/dashboard.txt new file mode 100644 index 0000000..6003ba1 --- /dev/null +++ b/evidence/native-sandbox/dashboard.txt @@ -0,0 +1,36 @@ +# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels +# date: 2026-11-01 00:23:49, commit: unknown +# command: python student-tests/check_dashboard.py +# exit code: 0, 0.5 s + +PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident) + empty now: CPU контейнеров / лимит + empty now: Collector: потери и отказы + empty now: SDK приложений: экспорт и потери + empty now: alerts.yml: MessengerDbPoolSaturated + empty now: alerts.yml: MessengerHighErrorRate + empty now: alerts.yml: MessengerQueueLagGrowing + empty now: alerts.yml: MessengerWorkerNoConsumers + empty now: alerts.yml: TelemetryCollectorDown + empty now: alerts.yml: TelemetryExportFailing + empty now: Алерты Prometheus (pending/firing) + empty now: Память: RSS процесса и контейнер +LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error" +TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 5 trace(s) + +data: http_server_request_duration_seconds_count{job="messenger-api"} 39 series <- HTTP requests/duration (3) +data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3) +data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4) +data: messenger_queue_depth 2 series <- queue depth (4) +data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4) +data: db_client_connection_count 4 series <- DB pool used/idle (4) +data: db_client_connection_pending_requests 2 series <- DB pool waiting (4) +data: messenger_worker_busy 1 series <- worker busy (4) +data: messenger_worker_capacity 1 series <- worker capacity (4) +data: process_cpu_time_seconds_total 12 series <- process CPU (4) +data: process_memory_usage_bytes 6 series <- process RAM (4) +data: otelcol_exporter_queue_size 3 series <- collector queue (4) + +labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, error_type, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version +http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready'] +PASS: every dashboard/rule query runs, key signals have data, no forbidden labels diff --git a/evidence/native-sandbox/idempotency.txt b/evidence/native-sandbox/idempotency.txt new file mode 100644 index 0000000..c2efb08 --- /dev/null +++ b/evidence/native-sandbox/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-11-01 00:52:50, commit: unknown +# command: python student-tests/test_idempotency.py +# exit code: 0, 2.2 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'a1a1a1a1a1a1': 10, 'a2a2a2a2a2a2': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/native-sandbox/incident-collector-down.json b/evidence/native-sandbox/incident-collector-down.json new file mode 100644 index 0000000..2910ce1 --- /dev/null +++ b/evidence/native-sandbox/incident-collector-down.json @@ -0,0 +1,39 @@ +{ + "scenario": "collector-down", + "date": "2026-11-01 00:39:14", + "outage_s": 90, + "baseline": { + "requests": 1002, + "codes": { + "201": 1002 + }, + "success_ratio": 1.0, + "success_rps": 66.8, + "p95_ms_successful": 15.2 + }, + "during": { + "requests": 6263, + "codes": { + "201": 6263 + }, + "success_ratio": 1.0, + "success_rps": 67.4, + "p95_ms_successful": 14.3 + }, + "api_mib_before": 280.5, + "api_mib_max": 292.5, + "sdk_lost_spans_api": { + "processor_queue_full": 0, + "exporter_failed": 24576 + }, + "sdk_exported_ok_after_restart": 10524, + "after": { + "requests": 1703, + "codes": { + "201": 1703 + }, + "success_ratio": 1.0, + "success_rps": 68.1, + "p95_ms_successful": 13.8 + } +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-collector.txt b/evidence/native-sandbox/incident-collector.txt new file mode 100644 index 0000000..8a94682 --- /dev/null +++ b/evidence/native-sandbox/incident-collector.txt @@ -0,0 +1,22 @@ +# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted +# date: 2026-11-01 00:41:37, commit: unknown +# command: python student-tests/incidents.py collector-down +# exit code: 0, 143.3 s + +=== collector-down +[ 20.3s] baseline: {'requests': 1002, 'codes': {'201': 1002}, 'success_ratio': 1.0, 'success_rps': 66.8, 'p95_ms_successful': 15.2}, api containers 280.5 MiB +[ 20.6s] docker compose stop otel-collector +[ 20.8s] {'t': 0.3, 'api_mib': 283.0} +[ 31.0s] {'t': 10.5, 'api_mib': 283.2} +[ 41.3s] {'t': 20.8, 'api_mib': 288.6} +[ 51.5s] {'t': 31.0, 'api_mib': 289.0} +[ 61.7s] {'t': 41.2, 'api_mib': 289.7} +[ 71.9s] {'t': 51.4, 'api_mib': 292.5} +[ 82.1s] {'t': 61.6, 'api_mib': 292.4} +[ 92.4s] {'t': 71.9, 'api_mib': 292.3} +[ 102.6s] {'t': 82.1, 'api_mib': 291.2} +[ 113.2s] docker compose start otel-collector +[ 143.2s] API during collector outage: {'requests': 6263, 'codes': {'201': 6263}, 'success_ratio': 1.0, 'success_rps': 67.4, 'p95_ms_successful': 14.3}; api memory 280.5 -> max 292.5 MiB; SDK lost spans (api): {'processor_queue_full': 0, 'exporter_failed': 24576}; exported OK in the last minute after restart: 10524 +[ 143.2s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-collector-down.json +PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start +1 passed, 0 failed diff --git a/evidence/native-sandbox/incident-db-slow.json b/evidence/native-sandbox/incident-db-slow.json new file mode 100644 index 0000000..d73c0ac --- /dev/null +++ b/evidence/native-sandbox/incident-db-slow.json @@ -0,0 +1,82 @@ +{ + "scenario": "db-slow", + "date": "2026-11-01 00:42:49", + "lock_seconds": 210, + "baseline": { + "requests": 2648, + "codes": { + "201": 2648 + }, + "success_ratio": 1.0, + "success_rps": 132.4, + "p95_ms_successful": 18.5, + "server_p95_s": 0.0214 + }, + "firing_after_lock_s": 120.1, + "diagnosis": { + "http_5xx_ratio_1m": 1.0, + "http_p95_messages_s": 7.375, + "pool_used": 8.0, + "pool_wait_p95_s": 0.001, + "status_codes_1m": { + "201": 0.0, + "200": 0.0, + "404": 0.0, + "400": 0.0, + "409": 0.0, + "503": 1.6 + }, + "reads_during_lock_ms": { + "GET /users/me + GET /chats, avg per request": 4.7, + "max": 5.4 + }, + "api_log_db_unavailable_by_error_type": { + "QueryCanceled": 184 + }, + "error_trace": { + "trace_id": "1c7f8dd06427574a5f155558aa4cd975", + "server_ms": 5006.7, + "slowest_db_span": "INSERT", + "slowest_db_ms": 5000.6, + "db_span_status": "ERROR", + "db_share": 0.999, + "tree": [ + "- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503", + " - [api] redis GET (CLIENT, 0.2 ms)", + " - [api] SELECT (CLIENT, 0.4 ms)", + " - [api] INSERT (CLIENT, 5000.6 ms) ERROR" + ] + }, + "handler_cpu_contrast": { + "trace_id": "a6da2e3fe7b0c229773f36fa507293ca", + "server_ms": 34.0, + "argon2id_ms": 29.0, + "db_ms_total": 0.6, + "tree": [ + "- [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201", + " - [api] argon2id hash (INTERNAL, 29.0 ms)", + " - [api] INSERT (CLIENT, 0.6 ms)" + ] + } + }, + "during_lock": { + "requests": 326, + "codes": { + "503": 320, + "201": 6 + }, + "success_ratio": 0.0184, + "success_rps": 0.0, + "p95_ms_successful": 2718.4 + }, + "resolved_after_unlock_s": 30.0, + "after": { + "requests": 3298, + "codes": { + "201": 3298 + }, + "success_ratio": 1.0, + "success_rps": 131.9, + "p95_ms_successful": 21.0 + } +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-db.txt b/evidence/native-sandbox/incident-db.txt new file mode 100644 index 0000000..b0471c0 --- /dev/null +++ b/evidence/native-sandbox/incident-db.txt @@ -0,0 +1,56 @@ +# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved +# date: 2026-11-01 00:47:15, commit: unknown +# command: python student-tests/incidents.py db-slow +# exit code: 0, 265.4 s + +=== db-slow +[ 0.1s] load: 8 threads POST /messages +[ 25.1s] baseline: {'requests': 2648, 'codes': {'201': 2648}, 'success_ratio': 1.0, 'success_rps': 132.4, 'p95_ms_successful': 18.5, 'server_p95_s': 0.0214} +[ 25.1s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait) +[ 25.1s] 5xx ratio=0.0 p95=0.021 s pool used=0.0 waiting=0.0 +[ 30.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 35.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 40.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 45.1s] 5xx ratio=0.005 p95=0.022 s pool used=8.0 waiting=0.0 +[ 50.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0 +[ 55.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0 +[ 60.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0 +[ 65.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0 +[ 70.1s] 5xx ratio=0.01 p95=5.688 s pool used=8.0 waiting=0.0 +[ 75.2s] 5xx ratio=0.069 p95=5.688 s pool used=8.0 waiting=0.0 +[ 80.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0 +[ 85.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0 +[ 90.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 95.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 100.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 105.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 110.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 115.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 120.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 125.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 130.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 135.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 140.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 145.2s] ALERT MessengerHighErrorRate firing (120.1 s after lock) +[ 151.5s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 4.7, "max": 5.4}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 184}} +[ 151.5s] error trace (DB): {"trace_id": "1c7f8dd06427574a5f155558aa4cd975", "server_ms": 5006.7, "slowest_db_span": "INSERT", "slowest_db_ms": 5000.6, "db_span_status": "ERROR", "db_share": 0.999} + - [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.4 ms) + - [api] INSERT (CLIENT, 5000.6 ms) ERROR +[ 151.5s] contrast, slow handler (CPU): {"trace_id": "a6da2e3fe7b0c229773f36fa507293ca", "server_ms": 34.0, "argon2id_ms": 29.0, "db_ms_total": 0.6} + - [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201 + - [api] argon2id hash (INTERNAL, 29.0 ms) + - [api] INSERT (CLIENT, 0.6 ms) +[ 235.4s] lock released (psql exit 0), 210.3 s after LOCK +[ 235.4s] ... MessengerHighErrorRate: firing (0 s) +[ 240.4s] ... MessengerHighErrorRate: firing (5 s) +[ 245.4s] ... MessengerHighErrorRate: firing (10 s) +[ 250.4s] ... MessengerHighErrorRate: firing (15 s) +[ 255.4s] ... MessengerHighErrorRate: firing (20 s) +[ 260.4s] ... MessengerHighErrorRate: firing (25 s) +[ 265.4s] ALERT MessengerHighErrorRate resolved 30.0 s after unlock +[ 265.4s] load: during lock {'requests': 326, 'codes': {'503': 320, '201': 6}, 'success_ratio': 0.0184, 'success_rps': 0.0, 'p95_ms_successful': 2718.4}; after {'requests': 3298, 'codes': {'201': 3298}, 'success_ratio': 1.0, 'success_rps': 131.9, 'p95_ms_successful': 21.0} +[ 265.4s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-db-slow.json +PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved +1 passed, 0 failed diff --git a/evidence/native-sandbox/incident-telemetry-down.json b/evidence/native-sandbox/incident-telemetry-down.json new file mode 100644 index 0000000..9e2683d --- /dev/null +++ b/evidence/native-sandbox/incident-telemetry-down.json @@ -0,0 +1,307 @@ +{ + "scenario": "telemetry-down", + "date": "2026-11-01 00:32:59", + "stopped": [ + "tempo", + "loki" + ], + "outage_s": 150, + "baseline": { + "requests": 955, + "codes": { + "201": 955 + }, + "success_ratio": 1.0, + "success_rps": 63.7, + "p95_ms_successful": 17.9 + }, + "collector_heap_before_mib": 44.1, + "collector_container_before_mib": 224.4, + "during": { + "requests": 11500, + "codes": { + "201": 11500 + }, + "success_ratio": 1.0, + "success_rps": 68.3, + "p95_ms_successful": 13.2 + }, + "samples": [ + { + "t": 0.0, + "collector_heap_mib": 39.8, + "collector_rss_mib": 228.5, + "collector_container_mib": 228.6, + "queue": { + "otlp_http/loki": 0.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 6.0 + }, + "alert": "inactive" + }, + { + "t": 10.1, + "collector_heap_mib": 52.2, + "collector_rss_mib": 228.6, + "collector_container_mib": 228.6, + "queue": { + "otlp_http/loki": 5.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 10.0 + }, + "alert": "inactive" + }, + { + "t": 20.3, + "collector_heap_mib": 52.2, + "collector_rss_mib": 228.6, + "collector_container_mib": 228.7, + "queue": { + "otlp_http/loki": 5.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 10.0 + }, + "alert": "inactive" + }, + { + "t": 30.4, + "collector_heap_mib": 63.3, + "collector_rss_mib": 230.2, + "collector_container_mib": 233.6, + "queue": { + "otlp_http/loki": 12.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 14.0 + }, + "alert": "inactive" + }, + { + "t": 40.6, + "collector_heap_mib": 48.5, + "collector_rss_mib": 234.5, + "collector_container_mib": 234.5, + "queue": { + "otlp_http/loki": 20.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 18.0 + }, + "alert": "pending" + }, + { + "t": 50.7, + "collector_heap_mib": 48.5, + "collector_rss_mib": 234.5, + "collector_container_mib": 234.6, + "queue": { + "otlp_http/loki": 20.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 18.0 + }, + "alert": "pending" + }, + { + "t": 60.8, + "collector_heap_mib": 59.5, + "collector_rss_mib": 234.6, + "collector_container_mib": 234.9, + "queue": { + "otlp_http/loki": 27.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 22.0 + }, + "alert": "pending" + }, + { + "t": 71.0, + "collector_heap_mib": 70.7, + "collector_rss_mib": 239.4, + "collector_container_mib": 242.2, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 26.0 + }, + "alert": "pending" + }, + { + "t": 81.2, + "collector_heap_mib": 70.7, + "collector_rss_mib": 239.4, + "collector_container_mib": 247.4, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 26.0 + }, + "alert": "pending" + }, + { + "t": 91.3, + "collector_heap_mib": 58.1, + "collector_rss_mib": 247.4, + "collector_container_mib": 247.4, + "queue": { + "otlp_http/loki": 42.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 30.0 + }, + "alert": "pending" + }, + { + "t": 101.4, + "collector_heap_mib": 69.3, + "collector_rss_mib": 247.5, + "collector_container_mib": 247.5, + "queue": { + "otlp_http/loki": 50.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 32.0 + }, + "alert": "firing" + }, + { + "t": 111.6, + "collector_heap_mib": 69.3, + "collector_rss_mib": 247.5, + "collector_container_mib": 249.7, + "queue": { + "otlp_http/loki": 50.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 32.0 + }, + "alert": "firing" + }, + { + "t": 121.8, + "collector_heap_mib": 80.2, + "collector_rss_mib": 249.9, + "collector_container_mib": 258.8, + "queue": { + "otlp_http/loki": 57.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 36.0 + }, + "alert": "firing" + }, + { + "t": 131.9, + "collector_heap_mib": 92.7, + "collector_rss_mib": 262.3, + "collector_container_mib": 265.4, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 40.0 + }, + "alert": "firing" + }, + { + "t": 142.1, + "collector_heap_mib": 92.7, + "collector_rss_mib": 262.3, + "collector_container_mib": 273.3, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 40.0 + }, + "alert": "firing" + } + ], + "collector_heap_max_mib": 92.7, + "collector_rss_max_mib": 262.3, + "collector_container_max_mib": 273.3, + "TelemetryExportFailing_firing_at_s": 101.4, + "queue_drained_after_start_s": 80.0, + "probe_traces": [ + { + "sent_at_s": 0.0, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 10.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 20.3, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 30.4, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 40.6, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 50.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 60.8, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 71.0, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 81.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 91.3, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 101.4, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 111.6, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 121.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 131.9, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 142.1, + "http": 201, + "trace_in_tempo": false + } + ], + "lost_in_collector": { + "spans": 4096, + "log_records": 269 + }, + "resolved_after_start_s": 155.4, + "after": { + "requests": 10186, + "codes": { + "201": 10186 + }, + "success_ratio": 1.0, + "success_rps": 67.7, + "p95_ms_successful": 14.2 + } +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-telemetry.txt b/evidence/native-sandbox/incident-telemetry.txt new file mode 100644 index 0000000..8add4c3 --- /dev/null +++ b/evidence/native-sandbox/incident-telemetry.txt @@ -0,0 +1,41 @@ +# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted +# date: 2026-11-01 00:39:14, commit: unknown +# command: python student-tests/incidents.py telemetry-down +# exit code: 0, 374.8 s + +=== telemetry-down +[ 20.2s] baseline: {'requests': 955, 'codes': {'201': 955}, 'success_ratio': 1.0, 'success_rps': 63.7, 'p95_ms_successful': 17.9}, collector heap 44.1 MiB, container 224.4 MiB +[ 50.9s] docker compose stop tempo loki: stopped +[ 51.1s] {'t': 0.0, 'collector_heap_mib': 39.8, 'collector_rss_mib': 228.5, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 0.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 6.0}, 'alert': 'inactive'} +[ 61.2s] {'t': 10.1, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'} +[ 71.3s] {'t': 20.3, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.7, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'} +[ 81.5s] {'t': 30.4, 'collector_heap_mib': 63.3, 'collector_rss_mib': 230.2, 'collector_container_mib': 233.6, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 14.0}, 'alert': 'inactive'} +[ 91.6s] {'t': 40.6, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.5, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'} +[ 101.8s] {'t': 50.7, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.6, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'} +[ 112.0s] {'t': 60.8, 'collector_heap_mib': 59.5, 'collector_rss_mib': 234.6, 'collector_container_mib': 234.9, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'pending'} +[ 122.1s] {'t': 71.0, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 242.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'} +[ 132.2s] {'t': 81.2, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'} +[ 142.4s] {'t': 91.3, 'collector_heap_mib': 58.1, 'collector_rss_mib': 247.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 30.0}, 'alert': 'pending'} +[ 152.5s] {'t': 101.4, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 247.5, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'} +[ 162.7s] {'t': 111.6, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 249.7, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'} +[ 172.9s] {'t': 121.8, 'collector_heap_mib': 80.2, 'collector_rss_mib': 249.9, 'collector_container_mib': 258.8, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 36.0}, 'alert': 'firing'} +[ 183.0s] {'t': 131.9, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 265.4, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'} +[ 193.2s] {'t': 142.1, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 273.3, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'} +[ 219.3s] docker compose start tempo loki +[ 299.3s] exporter queues drained 80.0 s after start +[ 324.7s] ... TelemetryExportFailing: firing (0 s) +[ 329.7s] ... TelemetryExportFailing: firing (5 s) +[ 334.7s] ... TelemetryExportFailing: firing (10 s) +[ 339.7s] ... TelemetryExportFailing: firing (15 s) +[ 344.7s] ... TelemetryExportFailing: firing (20 s) +[ 349.7s] ... TelemetryExportFailing: firing (25 s) +[ 354.7s] ... TelemetryExportFailing: firing (30 s) +[ 359.7s] ... TelemetryExportFailing: firing (35 s) +[ 364.7s] ... TelemetryExportFailing: firing (40 s) +[ 369.7s] ... TelemetryExportFailing: firing (45 s) +[ 374.7s] API during outage: {'requests': 11500, 'codes': {'201': 11500}, 'success_ratio': 1.0, 'success_rps': 68.3, 'p95_ms_successful': 13.2} +[ 374.7s] probe traces: 14/15 reached Tempo (lost: sent at [142.1] s) +[ 374.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 269}; collector heap 44.1 -> max 92.7 MiB (memory_limiter 200 MiB), container max 273.3 MiB +[ 374.7s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-telemetry-down.json +PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved +1 passed, 0 failed diff --git a/evidence/native-sandbox/incident-worker-stop.json b/evidence/native-sandbox/incident-worker-stop.json new file mode 100644 index 0000000..fe3a2fa --- /dev/null +++ b/evidence/native-sandbox/incident-worker-stop.json @@ -0,0 +1,49 @@ +{ + "scenario": "worker-stop", + "date": "2026-11-01 00:26:35", + "firing_after_stop_s": { + "MessengerWorkerNoConsumers": 66.4, + "MessengerQueueLagGrowing": 111.4 + }, + "diagnosis": { + "queue_consumers": 0.0, + "broker_queue_depth": 3.0, + "jobs_backlog_queued": 3.0, + "oldest_job_age_s": 97.3, + "outbox_oldest_age_s": 0.0, + "published_ok_last_5m": 3.102218085931441, + "worker_metrics_last_seen_s": 110.0, + "last_worker_log": { + "event": "service.stopped", + "message": "stopped" + }, + "upload_trace_span_names": [ + "INSERT", + "NOTIFY", + "POST /api/v1/chats/{chat_id}/attachments", + "S3 PutObject", + "SELECT", + "UPDATE", + "create attachments", + "inspect image", + "publish attachments", + "redis GET" + ], + "upload_trace_has_consumer_span": false + }, + "ready_after_start_s": [ + 1.5, + 1.6, + 2.0 + ], + "resolved_after_start_s": { + "MessengerWorkerNoConsumers": 17.0, + "MessengerQueueLagGrowing": 17.0 + }, + "after_recovery": { + "jobs_backlog_queued": 0.0, + "oldest_job_age_s": 0.0, + "queue_consumers": 2.0 + }, + "time_to_ready_p95_5m_s": 117.0 +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-worker.txt b/evidence/native-sandbox/incident-worker.txt new file mode 100644 index 0000000..88166d8 --- /dev/null +++ b/evidence/native-sandbox/incident-worker.txt @@ -0,0 +1,43 @@ +# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved +# date: 2026-11-01 00:28:44, commit: unknown +# command: python student-tests/incidents.py worker-stop +# exit code: 0, 128.6 s + +=== worker-stop +[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s +[ 1.3s] docker compose stop worker +[ 1.4s] uploaded 3 attachments: 202 queued (API accepts without worker) +[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {} +[ 6.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 16.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 21.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 26.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 31.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 36.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 41.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 46.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 51.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 56.4s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 61.5s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 66.5s] ALERT MessengerWorkerNoConsumers firing (66.4 s after stop) +[ 66.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 71.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 76.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 81.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 86.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 91.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 96.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 101.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 106.5s] consumers=0.0 backlog queued=3.0 oldest age=97.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 111.5s] ALERT MessengerQueueLagGrowing firing (111.4 s after stop) +[ 111.5s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 3.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 97.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 3.102218085931441, "worker_metrics_last_seen_s": 110.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET"], "upload_trace_has_consumer_span": false} +[ 111.6s] docker compose start worker +[ 113.5s] all attachments ready 2.0 s after worker start; backlog now 3.0 +[ 113.5s] ... MessengerWorkerNoConsumers: firing (0 s) +[ 118.5s] ... MessengerWorkerNoConsumers: firing (5 s) +[ 123.5s] ... MessengerWorkerNoConsumers: firing (10 s) +[ 128.5s] alerts resolved: {'MessengerWorkerNoConsumers': 17.0, 'MessengerQueueLagGrowing': 17.0}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0} +[ 128.5s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-worker-stop.json +PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-concurrency.txt b/evidence/native-sandbox/jobs-concurrency.txt new file mode 100644 index 0000000..f5f5735 --- /dev/null +++ b/evidence/native-sandbox/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-11-01 00:52:34, commit: unknown +# command: python student-tests/test_jobs.py concurrency +# exit code: 0, 7.1 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 4.1 s after worker start, peak processing 2 of 8 (22 samples) +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-outbox.txt b/evidence/native-sandbox/jobs-outbox.txt new file mode 100644 index 0000000..68b16d5 --- /dev/null +++ b/evidence/native-sandbox/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-11-01 00:51:38, commit: unknown +# command: python student-tests/test_jobs.py outbox-crash +# exit code: 0, 7.4 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 18ab06f7-657b-4508-9ba5-844bd2c7ba2e ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-recovery.txt b/evidence/native-sandbox/jobs-recovery.txt new file mode 100644 index 0000000..41b8503 --- /dev/null +++ b/evidence/native-sandbox/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-11-01 00:48:13, commit: unknown +# command: python student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 44.2 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment 748b0b56-e355-48a0-9ac6-4b731a3432a9 +message with attachment 201, content 409, other messages 201; job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job 5784e1e7-7173-4a27-a3ad-72216f4a43a9 +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-retry.txt b/evidence/native-sandbox/jobs-retry.txt new file mode 100644 index 0000000..83f800a --- /dev/null +++ b/evidence/native-sandbox/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-11-01 00:52:27, commit: unknown +# command: python student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 49.1 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.40 s and 3.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id fce6904a-fe4b-429c-821a-e3b55a43b0ef attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] +4 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-variants.txt b/evidence/native-sandbox/jobs-variants.txt new file mode 100644 index 0000000..691301a --- /dev/null +++ b/evidence/native-sandbox/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-11-01 00:47:29, commit: unknown +# command: python student-tests/test_jobs.py variants +# exit code: 0, 14.3 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/native-sandbox/load.json b/evidence/native-sandbox/load.json new file mode 100644 index 0000000..48e138d --- /dev/null +++ b/evidence/native-sandbox/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.083, + "concurrency": 4, + "successful_requests": 8762, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 291.265, + "successful_latency_ms": { + "p50": 11.557, + "p95": 26.659, + "p99": 56.425 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/load.txt b/evidence/native-sandbox/load.txt new file mode 100644 index 0000000..5ecdade --- /dev/null +++ b/evidence/native-sandbox/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-11-01 00:42:08, commit: unknown +# command: python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.4 s + +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.083, + "concurrency": 4, + "successful_requests": 8762, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 291.265, + "successful_latency_ms": { + "p50": 11.557, + "p95": 26.659, + "p99": 56.425 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/race.txt b/evidence/native-sandbox/race.txt new file mode 100644 index 0000000..587dc51 --- /dev/null +++ b/evidence/native-sandbox/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-11-01 00:52:53, commit: unknown +# command: python student-tests/test_race.py +# exit code: 0, 1.5 s + +sent 100 messages from 10 clients in 0.34 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/native-sandbox/smoke.txt b/evidence/native-sandbox/smoke.txt new file mode 100644 index 0000000..dd1fccd --- /dev/null +++ b/evidence/native-sandbox/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment +# date: 2026-11-01 00:22:30, commit: unknown +# command: python tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 1.7 s + +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/native-sandbox/student-tests.txt b/evidence/native-sandbox/student-tests.txt new file mode 100644 index 0000000..a356d19 --- /dev/null +++ b/evidence/native-sandbox/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-11-01 00:52:52, commit: unknown +# command: python student-tests/test_api.py +# exit code: 0, 2.0 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/native-sandbox/summary.md b/evidence/native-sandbox/summary.md new file mode 100644 index 0000000..e53e6a2 --- /dev/null +++ b/evidence/native-sandbox/summary.md @@ -0,0 +1,48 @@ +# Сводка проверок (нативная песочница, без Docker) + +Стенд теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, dispatcher, worker, janitor (Python 3.11, requirements.txt), otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из infra/ (адреса -> 127.0.0.1). Команды `docker compose` тестов выполнялись через подмену на управление процессами. Машина: Linux x86_64, 2 vCPU, 8 GiB RAM. + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt | +| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 2 | evidence/native-sandbox/smoke.txt | +| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 1 | evidence/native-sandbox/configs.txt | +| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 77 | evidence/native-sandbox/telemetry.txt | +| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 0 | evidence/native-sandbox/dashboard.txt | +| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 52 | evidence/native-sandbox/telemetry-history.txt | +| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 129 | evidence/native-sandbox/incident-worker.txt | +| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 265 | evidence/native-sandbox/incident-db.txt | +| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 375 | evidence/native-sandbox/incident-telemetry.txt | +| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 143 | evidence/native-sandbox/incident-collector.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 14 | evidence/native-sandbox/jobs-variants.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 44 | evidence/native-sandbox/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 7 | evidence/native-sandbox/jobs-outbox.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 14 | evidence/native-sandbox/attachments.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/native-sandbox/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 2 | evidence/native-sandbox/student-tests.txt | +| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/native-sandbox/load.txt | + +# Замеры + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с | +| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) | + +Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной). diff --git a/evidence/native-sandbox/telemetry-history.txt b/evidence/native-sandbox/telemetry-history.txt new file mode 100644 index 0000000..381a337 --- /dev/null +++ b/evidence/native-sandbox/telemetry-history.txt @@ -0,0 +1,9 @@ +# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) +# date: 2026-11-01 00:26:35, commit: unknown +# command: python student-tests/test_telemetry.py history +# exit code: 0, 52.2 s + +before restart: trace 52668075eebc7fd4938de9690ba2aad1 found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 719 +after restart of prometheus, loki, tempo: trace 52668075eebc7fd4938de9690ba2aad1 (4 spans), 1 log record(s), metric at t-30s = 719 (same value) +PASS: persistent volumes keep traces, logs and metrics across restarts of the stores +1 passed, 0 failed diff --git a/evidence/native-sandbox/telemetry.txt b/evidence/native-sandbox/telemetry.txt new file mode 100644 index 0000000..bcc0d51 --- /dev/null +++ b/evidence/native-sandbox/telemetry.txt @@ -0,0 +1,145 @@ +# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets +# date: 2026-11-01 00:23:48, commit: unknown +# command: python student-tests/test_telemetry.py request upload-chain retry-chain replay-link +# exit code: 0, 77.1 s + +POST message: 201, X-Request-Id=4d281ed1-39de-484a-add2-8480ef33ab9c, trace_id=97ebaba763fec926fa1e1ed3fc0fae7a (client traceparent) +- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5.8 ms) http.response.status_code=201 + - [api] redis GET (CLIENT, 0.3 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] INSERT (CLIENT, 0.8 ms) +Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (5.9 ms)' event=http.request severity=INFO service=messenger-api trace_id=97ebaba763fec926fa1e1ed3fc0fae7a span_id=82ab66080cb50d2a request_id=4d281ed1-39de-484a-add2-8480ef33ab9c +PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 64.6 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.4 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] inspect image (INTERNAL, 45.7 ms) + - [api] INSERT (CLIENT, 1.1 ms) + - [api] S3 PutObject (CLIENT, 5.1 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1 + - [api] UPDATE (CLIENT, 0.9 ms) + - [api] create attachments (PRODUCER, 3.5 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.9 ms) + - [api] INSERT (CLIENT, 0.9 ms) + - [api] INSERT (CLIENT, 0.8 ms) + - [api] NOTIFY (CLIENT, 0.3 ms) + - [dispatcher] publish attachments (PRODUCER, 4.4 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.9 ms) + - [worker] process attachments.process.v1 (CONSUMER, 280.0 ms) messenger.job.attempt=1 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 1.4 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 1.4 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] S3 GetObject (CLIENT, 2.9 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1 + - [worker] verify source sha256 (INTERNAL, 0.1 ms) + - [worker] imaging (INTERNAL, 238.2 ms) + - [worker] S3 PutObject (CLIENT, 4.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/original + - [worker] S3 PutObject (CLIENT, 3.4 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/crop.jpg + - [worker] commit result (INTERNAL, 7.5 ms) + - [worker] SELECT (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 1.0 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] UPDATE (CLIENT, 1.0 ms) + - [worker] UPDATE (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] cleanup staging (INTERNAL, 4.1 ms) + - [worker] S3 DeleteObject (CLIENT, 2.4 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1 + - [worker] UPDATE (CLIENT, 1.4 ms) +worker logs with trace_id 0eecc096d0191a19e41ad1547110183c: ['job.claimed', 'job.ready'] +PASS: one trace 0eecc096d0191a19e41ad1547110183c: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker) +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 17.4 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.3 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] inspect image (INTERNAL, 0.3 ms) + - [api] INSERT (CLIENT, 0.7 ms) + - [api] S3 PutObject (CLIENT, 5.0 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [api] UPDATE (CLIENT, 0.6 ms) + - [api] create attachments (PRODUCER, 2.9 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.6 ms) + - [api] INSERT (CLIENT, 0.5 ms) + - [api] INSERT (CLIENT, 0.5 ms) + - [api] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 3.2 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.5 ms) + - [worker] process attachments.process.v1 (CONSUMER, 255.7 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable + - [worker] SELECT (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] S3 GetObject (CLIENT, 3.4 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 238.9 ms) + - [worker] SELECT (CLIENT, 0.6 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] create attachments (PRODUCER, 1.5 ms) messenger.job.attempt=2 + - [worker] INSERT (CLIENT, 0.8 ms) + - [worker] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 3.3 ms) messenger.job.attempt=2 + - [dispatcher] UPDATE (CLIENT, 0.6 ms) + - [worker] process attachments.process.v1 (CONSUMER, 284.4 ms) messenger.job.attempt=2 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.7 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] S3 GetObject (CLIENT, 3.5 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [worker] verify source sha256 (INTERNAL, 0.1 ms) + - [worker] imaging (INTERNAL, 250.7 ms) + - [worker] S3 PutObject (CLIENT, 4.7 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/original + - [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 3.0 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/crop.jpg + - [worker] commit result (INTERNAL, 6.6 ms) + - [worker] SELECT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.5 ms) + - [worker] cleanup staging (INTERNAL, 4.6 ms) + - [worker] S3 DeleteObject (CLIENT, 2.7 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [worker] UPDATE (CLIENT, 1.3 ms) + - [worker] INSERT (CLIENT, 0.5 ms) +worker log: 'job 9a82f341-ec57-49e5-aa86-6263681529c1 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable +messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1 +PASS: retry in one trace 1abe27ebd519ac0854a844d18d679bf6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged +original trace 5b3ea90278cb5e18b6ad08cd32349c0e: 5 attempts, last outcome failed +- [ops] ops replay (INTERNAL, 16.5 ms) + - [ops] SELECT (CLIENT, 2.3 ms) + - [ops] UPDATE (CLIENT, 0.9 ms) + - [ops] UPDATE (CLIENT, 0.8 ms) + - [ops] INSERT (CLIENT, 0.7 ms) + - [ops] create attachments (PRODUCER, 1.1 ms) links=1 messenger.job.attempt=6 + - [ops] INSERT (CLIENT, 0.7 ms) + - [ops] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 5.3 ms) messenger.job.attempt=6 + - [worker] process attachments.process.v1 (CONSUMER, 281.7 ms) messenger.job.attempt=6 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] S3 GetObject (CLIENT, 2.7 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00 + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 251.3 ms) + - [worker] S3 PutObject (CLIENT, 3.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/original + - [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/crop.jpg + - [worker] commit result (INTERNAL, 6.0 ms) + - [worker] SELECT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] UPDATE (CLIENT, 0.7 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] cleanup staging (INTERNAL, 5.9 ms) + - [worker] S3 DeleteObject (CLIENT, 3.3 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00 + - [worker] UPDATE (CLIENT, 2.1 ms) + - [dispatcher] UPDATE (CLIENT, 1.3 ms) +replay trace 6d0327f64ceb5758c4a19b30aceed617: create span links -> ['5b3ea90278cb5e18b6ad08cd32349c0e'] (original upload trace 5b3ea90278cb5e18b6ad08cd32349c0e) +PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it +4 passed, 0 failed diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..61f5154 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,40 @@ +# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) +# date: 2026-11-01 20:46:08, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 62.0 s + +before: container, ID, started at + messenger-lab6-api-1 171a473b0d28 2026-11-01T17:43:16.63584434Z + messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:43:36.466962173Z + messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:31:14.251298706Z + messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:06:55.450862118Z + messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:07:17.636225869Z + messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:07:20.27137747Z + messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:22:20.690589644Z + messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:27:23.730596273Z + messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:04.263026279Z + messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:09:18.447414724Z + messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:30:47.892983401Z + messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:44:43.823155102Z + messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:40:41.919464505Z + messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:22:20.681510324Z + messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:38:33.102546081Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 60.9 s): container, ID, started at, restarted? + messenger-lab6-api-1 171a473b0d28 2026-11-01T17:46:02.488153329Z same container: True, restarted: True + messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:46:02.317979748Z same container: True, restarted: True + messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:46:00.495060123Z same container: True, restarted: True + messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:45:57.449417453Z same container: True, restarted: True + messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:45:57.621731679Z same container: True, restarted: True + messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:46:05.984667885Z same container: True, restarted: True + messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:45:21.206788727Z same container: True, restarted: True + messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:45:56.829306342Z same container: True, restarted: True + messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:19.869545292Z same container: True, restarted: True + messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:45:20.880337926Z same container: True, restarted: True + messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:45:21.007029551Z same container: True, restarted: True + messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:45:19.748047785Z same container: True, restarted: True + messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:45:29.752000764Z same container: True, restarted: True + messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:45:50.068926735Z same container: True, restarted: True + messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:46:00.664535867Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, s3, rabbitmq, api, janitor, dispatcher, worker, lb, otel-collector, prometheus, loki, tempo, grafana (all api replicas; same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 0000000..fb910cf --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-11-01 20:42:41, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.2 s + +sent 100 messages from 10 clients in 0.30 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 0000000..c772f3a --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-11-01 20:42:56, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 3.2 s + +session key: session:0fe0e5550e07d95d...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 147 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 0000000..2f38cc2 --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,53 @@ +# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap +# date: 2026-11-01 20:38:59, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_s3_security.py +# exit code: 0, 10.5 s + +anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous head_object directly to S3 (from the data network) -> 403 +anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied +anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied +the same file through the API for a chat member -> 200 +PASS: the bucket is private: no anonymous read, list, write or policy read +API key: list buckets -> ['messenger-attachments'] +API key: get own object -> allowed +API key: put object -> allowed +API key: delete object -> allowed +API key: list bucket -> allowed +API key: create another bucket -> denied: AccessDenied +API key: make bucket public (put policy) -> denied: NotImplemented +API key: read bucket policy -> denied: NoSuchBucketPolicy +API key: delete bucket -> denied: AccessDenied +API key: change versioning -> denied: NotImplemented +PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied +compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand) +127.0.0.1:8333 from the host -> connection refused +127.0.0.1:9333 from the host -> connection refused +127.0.0.1:8888 from the host -> connection refused +127.0.0.1:19333 from the host -> connection refused +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of grafana: ['grafana_admin_password'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of loki: - +secrets of migrate: ['database_url'] +secrets of otel-collector: - +secrets of postgres: ['postgres_password'] +secrets of prometheus: - +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of telemetry-init: - +secrets of tempo: - +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init +s3-init: s3-init: bucket messenger-attachments exists +s3-init: s3-init: api key can put/get/list/delete objects of its bucket +s3-init: s3-init: api key cannot create buckets (AccessDenied) +s3-init: s3-init: bucket is private (anonymous list/get/put -> [403]) +after the repeated bootstrap: old file downloads, new upload works +PASS: bucket/user/policy bootstrap can be repeated safely +4 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..93d9450 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment +# date: 2026-11-01 20:07:47, commit: 470d9d6 + uncommitted changes +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 1.0 s + +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 0000000..f3f5ab4 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,66 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services +# date: 2026-11-01 20:07:46, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/check_stack.py +# exit code: 0, 3.1 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api 171a473b0d28 Up 26 seconds (healthy) 8080/tcp +api 1db2fedff78f Up 26 seconds (healthy) 8080/tcp +dispatcher 2f899202c4d1 Up 47 seconds (healthy) +grafana 23e7b8bb1b3a Up 49 seconds (healthy) 127.0.0.1:3000->3000/tcp +janitor 6dad6b402ed9 Up 26 seconds (healthy) +lb 0bffe8d8d7d2 Up 24 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +loki c62735c8baca Up 54 seconds 127.0.0.1:3100->3100/tcp +migrate dd9753ec4186 Exited (0) 48 seconds ago +otel-collector 8c416bdc0221 Up 55 seconds +postgres ced386ea9bbd Up 55 seconds (healthy) +prometheus 451bf5466e96 Up 55 seconds (healthy) 127.0.0.1:9090->9090/tcp +rabbitmq ec502659b532 Up 55 seconds (healthy) +redis 3c003ba9b32a Up 55 seconds (healthy) +s3 1e70a509e403 Up 55 seconds (healthy) +s3-init 64471229b155 Exited (0) 27 seconds ago +telemetry-init 809e227f3528 Exited (0) 55 seconds ago +tempo 3daf3848457f Up 54 seconds 127.0.0.1:3200->3200/tcp +worker cbd8138493bf Up 26 seconds (healthy) + +container image cpus memory host ports networks +messenger-lab6-api-1 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry +messenger-lab6-api-2 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry +messenger-lab6-dispatcher-1 messenger-lab6-dispatcher 0.25 128 MiB - data,telemetry +messenger-lab6-grafana-1 grafana/grafana:12.4.14 0.50 384 MiB 127.0.0.1:3000 ops,telemetry +messenger-lab6-janitor-1 messenger-lab6-janitor 0.25 192 MiB - data,telemetry +messenger-lab6-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab6-loki-1 grafana/loki:3.7.9 1.00 512 MiB 127.0.0.1:3100 ops,telemetry +messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.164.0 0.50 384 MiB - telemetry +messenger-lab6-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab6-prometheus-1 prom/prometheus:v3.15.1 1.00 512 MiB 127.0.0.1:9090 ops,telemetry +messenger-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data +messenger-lab6-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data +messenger-lab6-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data +messenger-lab6-tempo-1 grafana/tempo:3.1.1 1.00 512 MiB 127.0.0.1:3200 ops,telemetry +messenger-lab6-worker-1 messenger-lab6-worker 2.00 1536 MiB - data,telemetry + +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of grafana: ['grafana_admin_password'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of loki: - +secrets of migrate: ['database_url'] +secrets of otel-collector: - +secrets of postgres: ['postgres_password'] +secrets of prometheus: - +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of telemetry-init: - +secrets of tempo: - +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['171a473b0d28', '1db2fedff78f'] +/health/ready -> 200 from 1db2fedff78f +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/rabbitmq only in the internal data network, worker/dispatcher/janitor in data+telemetry, collector only in the internal telemetry network, prometheus/loki/tempo/grafana on 127.0.0.1 with persistent volumes, pinned images, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000..67775d3 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-11-01 20:42:40, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.8 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 0000000..72d4c17 --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,108 @@ +# Сводка проверок (Docker-стенд, 2026-11-01) + +Шаги запускались группами (`collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history`, затем каждый инцидент и `load` отдельно, затем `--from jobs-variants`); `load` выполнен дважды — в группе lab6 и в регрессии, evidence/load.txt и load.json — от второго прогона. + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/check.txt | +| stack | stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services | PASS | 3 | evidence/stack.txt | +| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 1 | evidence/smoke.txt | +| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 6 | evidence/configs.txt | +| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 79 | evidence/telemetry.txt | +| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 1 | evidence/dashboard.txt | +| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 39 | evidence/telemetry-history.txt | +| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 162 | evidence/incident-worker.txt | +| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 286 | evidence/incident-db.txt | +| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 380 | evidence/incident-telemetry.txt | +| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 156 | evidence/incident-collector.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt | +| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 73 | evidence/jobs-limits.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 17 | evidence/jobs-outbox.txt | +| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 157 | evidence/jobs-crash.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 78 | evidence/jobs-retry.txt | +| jobs-janitor-replay | lab5: janitor does not delete objects re-written by replay (review fix) | PASS | 187 | evidence/jobs-janitor-replay.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 16 | evidence/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt | +| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt | +| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 160 | evidence/consistency.txt | +| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 53 | evidence/upload-memory.txt | +| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt | +| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt | +| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 12 | evidence/db.txt | +| redis | Redis as the session store | PASS | 3 | evidence/redis.txt | +| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt | +| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 42 | evidence/failover.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| failures | Redis and PostgreSQL outages via the balancer | PASS | 56 | evidence/failures.txt | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 62 | evidence/persistence.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB. + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с | +| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) | + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 | +| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 | +| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 | +| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 | +| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | diff --git a/evidence/telemetry-history.txt b/evidence/telemetry-history.txt new file mode 100644 index 0000000..61abba2 --- /dev/null +++ b/evidence/telemetry-history.txt @@ -0,0 +1,9 @@ +# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) +# date: 2026-11-01 20:09:51, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_telemetry.py history +# exit code: 0, 38.6 s + +before restart: trace 0b9106c1057e9753186aec51b529946b found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 434 +after restart of prometheus, loki, tempo: trace 0b9106c1057e9753186aec51b529946b (4 spans), 1 log record(s), metric at t-30s = 434 (same value) +PASS: persistent volumes keep traces, logs and metrics across restarts of the stores +1 passed, 0 failed diff --git a/evidence/telemetry.txt b/evidence/telemetry.txt new file mode 100644 index 0000000..21e13af --- /dev/null +++ b/evidence/telemetry.txt @@ -0,0 +1,145 @@ +# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets +# date: 2026-11-01 20:09:11, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_telemetry.py request upload-chain retry-chain replay-link +# exit code: 0, 78.6 s + +POST message: 201, X-Request-Id=df174aef-1d0d-4340-b2d7-907241684f5e, trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce (client traceparent) +- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 4.6 ms) http.response.status_code=201 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.3 ms) + - [api] INSERT (CLIENT, 0.4 ms) +Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (4.6 ms)' event=http.request severity=INFO service=messenger-api trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce span_id=78cd8621fce4f3b8 request_id=df174aef-1d0d-4340-b2d7-907241684f5e +PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 13.5 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.4 ms) + - [api] inspect image (INTERNAL, 0.5 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238 + - [api] UPDATE (CLIENT, 0.7 ms) + - [api] create attachments (PRODUCER, 2.4 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.7 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] NOTIFY (CLIENT, 0.1 ms) + - [dispatcher] publish attachments (PRODUCER, 4.0 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.4 ms) + - [worker] process attachments.process.v1 (CONSUMER, 161.4 ms) messenger.job.attempt=1 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] S3 GetObject (CLIENT, 2.0 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238 + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 139.8 ms) + - [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/original + - [worker] S3 PutObject (CLIENT, 1.9 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 2.3 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/crop.jpg + - [worker] commit result (INTERNAL, 4.9 ms) + - [worker] SELECT (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] cleanup staging (INTERNAL, 4.7 ms) + - [worker] S3 DeleteObject (CLIENT, 2.2 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238 + - [worker] UPDATE (CLIENT, 2.2 ms) +worker logs with trace_id f5d1897588244cad5fc6ded83b930725: ['job.claimed', 'job.ready'] +PASS: one trace f5d1897588244cad5fc6ded83b930725: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker) +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 12.6 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] inspect image (INTERNAL, 0.3 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [api] UPDATE (CLIENT, 0.6 ms) + - [api] create attachments (PRODUCER, 1.4 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.4 ms) + - [api] INSERT (CLIENT, 0.3 ms) + - [api] INSERT (CLIENT, 0.3 ms) + - [api] NOTIFY (CLIENT, 0.1 ms) + - [dispatcher] publish attachments (PRODUCER, 4.2 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.4 ms) + - [worker] process attachments.process.v1 (CONSUMER, 135.6 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] S3 GetObject (CLIENT, 2.6 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 123.0 ms) + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] create attachments (PRODUCER, 1.0 ms) messenger.job.attempt=2 + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 5.2 ms) messenger.job.attempt=2 + - [dispatcher] UPDATE (CLIENT, 0.5 ms) + - [worker] process attachments.process.v1 (CONSUMER, 144.9 ms) messenger.job.attempt=2 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 125.7 ms) + - [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/original + - [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 1.7 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/crop.jpg + - [worker] commit result (INTERNAL, 4.4 ms) + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] cleanup staging (INTERNAL, 4.5 ms) + - [worker] S3 DeleteObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [worker] UPDATE (CLIENT, 2.5 ms) + - [worker] INSERT (CLIENT, 0.4 ms) +worker log: 'job 7bd96513-9f85-471e-b553-ec49f9fe4560 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable +messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1 +PASS: retry in one trace 5d34bb1eecbc90702d2f5e7de5b9bca6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged +original trace 38319800cdce5da47cb2b821dc03bf6e: 5 attempts, last outcome failed +- [ops] ops replay (INTERNAL, 12.3 ms) + - [ops] SELECT (CLIENT, 1.9 ms) + - [ops] UPDATE (CLIENT, 0.6 ms) + - [ops] UPDATE (CLIENT, 0.4 ms) + - [ops] INSERT (CLIENT, 0.5 ms) + - [ops] create attachments (PRODUCER, 0.8 ms) links=1 messenger.job.attempt=6 + - [ops] INSERT (CLIENT, 0.5 ms) + - [ops] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 70.0 ms) messenger.job.attempt=6 + - [dispatcher] UPDATE (CLIENT, 66.1 ms) + - [worker] process attachments.process.v1 (CONSUMER, 145.2 ms) messenger.job.attempt=6 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 125.7 ms) + - [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/original + - [worker] S3 PutObject (CLIENT, 2.0 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/crop.jpg + - [worker] commit result (INTERNAL, 4.4 ms) + - [worker] SELECT (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] cleanup staging (INTERNAL, 4.1 ms) + - [worker] S3 DeleteObject (CLIENT, 2.1 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f + - [worker] UPDATE (CLIENT, 1.8 ms) +replay trace 9417040b8f913c26894caaaf0996e442: create span links -> ['38319800cdce5da47cb2b821dc03bf6e'] (original upload trace 38319800cdce5da47cb2b821dc03bf6e) +PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it +4 passed, 0 failed diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..4a26755 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,124 @@ +{ + "date": "2026-11-01 20:42", + "api_limit_mib": 512, + "api_limit_cpus": 1.0, + "runs": [ + { + "scenario": "10 x ~9.5 MiB PNG (noise)", + "concurrency": 10, + "file_bytes": [ + 9953047 + ], + "total_mib": 94.9, + "all_202": true, + "wall_seconds": 1.95, + "latency_s": { + "p50": 1.77, + "max": 1.95 + }, + "sampler_rounds": 16, + "replicas": [ + { + "replica": "171a473b0d28", + "uploads_served": 5, + "rss_before_mib": 109.6, + "rss_peak_mib": 124.5, + "rss_growth_mib": 14.9, + "rss_peak_sampled_mib": 124.5, + "cgroup_peak_mib": 162.8, + "cgroup_peak_sampled_mib": 160.2 + }, + { + "replica": "1db2fedff78f", + "uploads_served": 5, + "rss_before_mib": 110.5, + "rss_peak_mib": 125.6, + "rss_growth_mib": 15.1, + "rss_peak_sampled_mib": 125.6, + "cgroup_peak_mib": 163.3, + "cgroup_peak_sampled_mib": 160.8 + } + ] + }, + { + "scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", + "concurrency": 10, + "file_bytes": [ + 116714 + ], + "total_mib": 1.1, + "all_202": true, + "wall_seconds": 1.3, + "latency_s": { + "p50": 0.98, + "max": 1.3 + }, + "sampler_rounds": 9, + "replicas": [ + { + "replica": "171a473b0d28", + "uploads_served": 5, + "rss_before_mib": 107.6, + "rss_peak_mib": 112.4, + "rss_growth_mib": 4.8, + "rss_peak_sampled_mib": 112.4, + "cgroup_peak_mib": 101.5, + "cgroup_peak_sampled_mib": 101.4 + }, + { + "replica": "1db2fedff78f", + "uploads_served": 5, + "rss_before_mib": 107.1, + "rss_peak_mib": 109.8, + "rss_growth_mib": 2.7, + "rss_peak_sampled_mib": 109.8, + "cgroup_peak_mib": 100.0, + "cgroup_peak_sampled_mib": 100.0 + } + ] + }, + { + "scenario": "20 mixed: large + 40 Mpx + JPEG", + "concurrency": 20, + "file_bytes": [ + 1492, + 116714, + 9953047 + ], + "total_mib": 133.2, + "all_202": true, + "wall_seconds": 1.95, + "latency_s": { + "p50": 1.45, + "max": 1.95 + }, + "sampler_rounds": 11, + "replicas": [ + { + "replica": "171a473b0d28", + "uploads_served": 10, + "rss_before_mib": 109.0, + "rss_peak_mib": 136.3, + "rss_growth_mib": 27.3, + "rss_peak_sampled_mib": 136.3, + "cgroup_peak_mib": 191.4, + "cgroup_peak_sampled_mib": 191.4 + }, + { + "replica": "1db2fedff78f", + "uploads_served": 10, + "rss_before_mib": 110.6, + "rss_peak_mib": 135.0, + "rss_growth_mib": 24.3, + "rss_peak_sampled_mib": 135.0, + "cgroup_peak_mib": 172.8, + "cgroup_peak_sampled_mib": 171.4 + } + ] + } + ], + "single_file_extra_rss_mib": { + "naive": 161, + "streaming": 0 + } +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 0000000..c10597c --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,13 @@ +# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas +# date: 2026-11-01 20:42:33, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json +# exit code: 0, 53.4 s + +client: Python 3.11.1 on Windows 10 +preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... +large: 9953047 bytes; 40 Mpx: 116714 bytes +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.77, "max": 1.95}, "sampler_rounds": 16, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 109.6, "rss_peak_mib": 124.5, "rss_growth_mib": 14.9, "rss_peak_sampled_mib": 124.5, "cgroup_peak_mib": 162.8, "cgroup_peak_sampled_mib": 160.2}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 110.5, "rss_peak_mib": 125.6, "rss_growth_mib": 15.1, "rss_peak_sampled_mib": 125.6, "cgroup_peak_mib": 163.3, "cgroup_peak_sampled_mib": 160.8}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.3, "latency_s": {"p50": 0.98, "max": 1.3}, "sampler_rounds": 9, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 107.6, "rss_peak_mib": 112.4, "rss_growth_mib": 4.8, "rss_peak_sampled_mib": 112.4, "cgroup_peak_mib": 101.5, "cgroup_peak_sampled_mib": 101.4}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 107.1, "rss_peak_mib": 109.8, "rss_growth_mib": 2.7, "rss_peak_sampled_mib": 109.8, "cgroup_peak_mib": 100.0, "cgroup_peak_sampled_mib": 100.0}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.45, "max": 1.95}, "sampler_rounds": 11, "replicas": [{"replica": "171a473b0d28", "uploads_served": 10, "rss_before_mib": 109.0, "rss_peak_mib": 136.3, "rss_growth_mib": 27.3, "rss_peak_sampled_mib": 136.3, "cgroup_peak_mib": 191.4, "cgroup_peak_sampled_mib": 191.4}, {"replica": "1db2fedff78f", "uploads_served": 10, "rss_before_mib": 110.6, "rss_peak_mib": 135.0, "rss_growth_mib": 24.3, "rss_peak_sampled_mib": 135.0, "cgroup_peak_mib": 172.8, "cgroup_peak_sampled_mib": 171.4}]} +one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB +PASS: all uploads succeeded; largest RSS growth of one replica: 27.3 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/faults.py b/faults.py new file mode 100644 index 0000000..2bb92a7 --- /dev/null +++ b/faults.py @@ -0,0 +1,51 @@ +"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1). + +Тест создаёт в общем volume /faults файлы <точка>., по одному на срабатывание: + + docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1' + +Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса) +и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски. + +Точки: + dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish) + dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана + worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет + worker_crash_after_commit — COMMIT результата был, ack ещё нет + worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease + worker_s3_error — временная ошибка S3 при записи результата + imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения +""" +import logging +import os +import sys +from pathlib import Path + +import settings + +log = logging.getLogger("messenger.faults") + + +def hit(point: str) -> bool: + if not settings.FAULT_INJECTION: + return False + directory = Path(settings.FAULTS_DIR) + try: + candidates = sorted(directory.glob(point + ".*")) + except OSError: + return False + for path in candidates: + try: + path.unlink() + except FileNotFoundError: + continue # этот файл забрал другой процесс + log.warning("FAULT INJECTION: %s", point) + return True + return False + + +def crash(point: str) -> None: + """Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений.""" + if hit(point): + sys.stderr.flush() + os._exit(137) diff --git a/imaging.py b/imaging.py new file mode 100644 index 0000000..a08fbb0 --- /dev/null +++ b/imaging.py @@ -0,0 +1,209 @@ +"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами. + + python imaging.py + +Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя +ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память, +IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker +убивает его по таймауту JOB_TIMEOUT_SECONDS. +Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру. + +Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout. +Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти; +прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной. + +Правила вариантов (contracts/README.md): +- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз + (минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160. +- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются). +- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется. +Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное +отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job +пишет в те же ключи S3 те же байты. +""" +import json +import os +import resource +import sys +import time + + +def apply_limits() -> None: + """Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов.""" + memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024 + cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60")) + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024)) + + +if __name__ == "__main__": + apply_limits() + +from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом + +PIPELINE_VERSION = 1 +THUMB_BOX = 256 +CROP_SIZE = 128 +OPTIMIZED_BOX = 1280 +THUMB_QUALITY = 85 +CROP_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +EXIT_INVALID = 3 +EXIT_TOO_LARGE = 4 +EXIT_MEMORY = 5 + +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose) +ORIENTATION = { + 2: [Image.Transpose.FLIP_LEFT_RIGHT], + 3: [Image.Transpose.ROTATE_180], + 4: [Image.Transpose.FLIP_TOP_BOTTOM], + 5: [Image.Transpose.TRANSPOSE], + 6: [Image.Transpose.ROTATE_270], + 7: [Image.Transpose.TRANSVERSE], + 8: [Image.Transpose.ROTATE_90], +} + + +class TooLarge(Exception): + pass + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + """Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def orientation_of(image: Image.Image) -> int: + try: + value = image.getexif().get(0x0112, 1) + except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем + return 1 + return value if value in ORIENTATION else 1 + + +def normalize(image: Image.Image) -> Image.Image: + """Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность).""" + mode = image.mode + transparent = "transparency" in image.info + if mode in ("I;16", "I;16B", "I;16L", "I"): + image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L") + mode = "L" + if mode == "1": + return image.convert("L") + if mode in ("P", "PA"): + return image.convert("RGBA" if transparent or mode == "PA" else "RGB") + if mode in ("LA", "La", "RGBa"): + return image.convert("RGBA") + if mode in ("L", "RGB") and transparent: + return image.convert("RGBA") + if mode in ("L", "RGB", "RGBA"): + return image + return image.convert("RGB") # CMYK, YCbCr, LAB, ... + + +def on_white(image: Image.Image) -> Image.Image: + """Прозрачность -> белый фон (для JPEG).""" + if image.mode != "RGBA": + return image + background = Image.new("RGB", image.size, (255, 255, 255)) + background.paste(image, mask=image.getchannel("A")) + return background + + +def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict: + Image.MAX_IMAGE_PIXELS = max_pixels + with Image.open(source) as image: + if image.format != FORMATS.get(content_type): + raise ValueError("format does not match stored content type") + width, height = image.size + if width * height > max_pixels: + raise TooLarge() + orientation = orientation_of(image) + icc = image.info.get("icc_profile") + swap = orientation in (5, 6, 7, 8) + oriented = (height, width) if swap else (width, height) + + # Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128 + # не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной. + target = fit(*oriented, OPTIMIZED_BOX) + stored_target = (target[1], target[0]) if swap else target + if image.format == "JPEG": + image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает + image.load() + work = normalize(image) + if work.size != stored_target: + work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0) + elif work is image: + work = work.copy() + for step in ORIENTATION.get(orientation, []): + work = work.transpose(step) + + extra = {"icc_profile": icc} if icc else {} + results = [] + + optimized = work if work.mode != "L" else work.convert("RGB") + path = os.path.join(out_dir, "optimized.webp") + optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra) + results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width, + "height": optimized.height, "file": "optimized.webp"}) + + thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии + thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS) + thumb = on_white(thumb) + path = os.path.join(out_dir, "thumbnail.jpg") + thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra) + results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width, + "height": thumb.height, "file": "thumbnail.jpg"}) + + side = min(work.size) + left, top = (work.width - side) // 2, (work.height - side) // 2 + crop = work.crop((left, top, left + side, top + side)) + crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)) + path = os.path.join(out_dir, "crop.jpg") + crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra) + results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE, + "file": "crop.jpg"}) + + return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height, + "orientation": orientation}, "variants": results} + + +def injected_fault() -> None: + """Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1).""" + fault = os.environ.get("FAULT_IMAGING", "") + if fault == "hang": + time.sleep(3600) + elif fault == "invalid": + raise ValueError("injected invalid image") + elif fault == "oom": + blocks = [] + while True: + blocks.append(bytearray(64 * 1024 * 1024)) + + +def main() -> int: + source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) + try: + injected_fault() + result = process(source, out_dir, content_type, max_pixels) + except (TooLarge, Image.DecompressionBombError): + return EXIT_TOO_LARGE + except MemoryError: + return EXIT_MEMORY + except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError): + # Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу. + return EXIT_INVALID + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..b350588 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,2288 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — lab6 observability", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Обзор и SLO (окно SLO — 1 час)", + "id": 2, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "id": 3, + "title": "RPS API", + "description": "Бизнес-запросы API в секунду (без /health), все реплики.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_requests:rate1m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 4, + "title": "Доля 5xx (1m)", + "description": "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_error_ratio:rate1m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 5, + "title": "p95 API (5m)", + "description": "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_request_duration_seconds:p95_5m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 6, + "title": "Старейшее задание", + "description": "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process.v1\"})", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 7, + "title": "SLO-1 доступность", + "description": "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:ratio_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 8, + "title": "Бюджет ошибок SLO-1", + "description": "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:error_budget_remaining_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0 + }, + { + "color": "green", + "value": 0.5 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 9, + "title": "SLO-2 POST < 250 мс", + "description": "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_latency_messages:ratio_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 10, + "title": "SLO-3 ready ≤ 30 с", + "description": "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_time_to_ready:ratio_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "table", + "id": 11, + "title": "Алерты Prometheus (pending/firing)", + "description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "ALERTS{alertstate=~\"pending|firing\"}", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true + } + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + } + }, + { + "type": "row", + "title": "API: запросы, ошибки, задержка (RED)", + "id": 12, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 13, + "title": "RPS по маршрутам", + "description": "Шаблон маршрута, а не URL: метка ограниченной кардинальности.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "{{http_request_method}} {{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 14, + "title": "RPS по классу статуса", + "description": "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "{{http_response_status_class}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 15, + "title": "Доля 5xx", + "description": "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_response_status_class=\"5xx\"}[$__rate_interval])) or on() vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "5xx ratio", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 16, + "title": "Задержка API p50 / p95 / p99", + "description": "histogram_quantile по сумме buckets всех реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 17, + "title": "p95 по маршрутам", + "description": "Где именно медленно.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 18, + "title": "Запросы в обработке (по репликам)", + "description": "http.server.active_requests: насыщение реплик API.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (http_server_active_requests{job=\"messenger-api\"})", + "legendFormat": "{{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Фоновая обработка: очередь, worker, время до ready", + "id": 19, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 20, + "title": "Попытки заданий в секунду по исходу", + "description": "ready / retry / failed / stale / lease_lost (messenger.job.duration count).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 21, + "title": "Длительность попытки p50 / p95 / p99 (ready)", + "description": "От claim до COMMIT ready в worker.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"ready\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"ready\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"ready\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 22, + "title": "Время до ready p50 / p95 / p99", + "description": "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{job_type=\"attachment.process.v1\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{job_type=\"attachment.process.v1\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{job_type=\"attachment.process.v1\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 23, + "title": "Глубина очереди и backlog", + "description": "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (messaging_destination_name) (messenger_queue_depth)", + "legendFormat": "broker {{messaging_destination_name}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (status) (messenger_jobs_backlog)", + "legendFormat": "jobs {{status}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 24, + "title": "Возраст старейшего задания и outbox", + "description": "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process.v1\"})", + "legendFormat": "oldest queued job", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_outbox_oldest_age_seconds)", + "legendFormat": "oldest unpublished outbox row", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 25, + "title": "Доставки и публикации", + "description": "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", + "legendFormat": "consumed {{outcome}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", + "legendFormat": "published {{outcome}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Насыщение: пул БД, worker, CPU/RAM", + "id": 26, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 27, + "title": "Пул PostgreSQL API: занято / максимум / ждут", + "description": "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (db_client_connection_count{db_client_connection_state=\"used\"})", + "legendFormat": "used {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(db_client_connection_max)", + "legendFormat": "max per replica", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (db_client_connection_pending_requests)", + "legendFormat": "waiting {{instance}}", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 28, + "title": "Ожидание соединения из пула p95", + "description": "Время получения соединения из пула API (не время запроса).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95 {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 29, + "title": "Worker: занятость (busy / capacity)", + "description": "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)", + "legendFormat": "saturation", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 30, + "title": "CPU процессов (ядра)", + "description": "rate(process.cpu.time): ядер на процесс.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 53 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 31, + "title": "CPU контейнеров / лимит", + "description": "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 53 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 32, + "title": "Память: RSS процесса и контейнер", + "description": "process.memory.usage (RSS) и container.memory.usage (cgroup).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 53 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (job, instance) (process_memory_usage_bytes)", + "legendFormat": "rss {{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (job, instance) (container_memory_usage_bytes)", + "legendFormat": "cgroup {{job}} {{instance}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Конвейер телеметрии: Collector и SDK", + "id": 33, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 34, + "title": "Collector: принято и отправлено", + "description": "Спаны/записи логов/точки метрик в секунду.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 62 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", + "legendFormat": "accepted spans", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", + "legendFormat": "accepted logs", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", + "legendFormat": "accepted metric points", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", + "legendFormat": "sent spans {{exporter}}", + "refId": "D", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", + "legendFormat": "sent logs {{exporter}}", + "refId": "E", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 35, + "title": "Collector: потери и отказы", + "description": "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 62 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", + "legendFormat": "send failed spans {{exporter}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", + "legendFormat": "send failed logs {{exporter}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", + "legendFormat": "queue full spans {{exporter}}", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", + "legendFormat": "refused spans (memory_limiter)", + "refId": "D", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 36, + "title": "Очереди exporter и RSS Collector", + "description": "Заполнение sending_queue (доля ёмкости) и память процесса Collector.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 62 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", + "legendFormat": "queue {{exporter}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 37, + "title": "Collector RSS", + "description": "Ограничено memory_limiter (200 MiB) при недоступном хранилище.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(otelcol_process_memory_rss)", + "legendFormat": "collector rss", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 38, + "title": "SDK приложений: экспорт и потери", + "description": "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=\"\"}[$__rate_interval]))", + "legendFormat": "exported spans {{job}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "export failed spans {{job}} {{error_type}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "export failed logs {{job}} {{error_type}}", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "queue full spans {{job}}", + "refId": "D", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Логи и трассы", + "id": 39, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "panels": [] + }, + { + "type": "logs", + "id": 40, + "title": "Предупреждения и ошибки (Loki)", + "description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 79 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "queryType": "range", + "expr": "{service_name=~\"messenger-.*\"} | detected_level=~\"warn|error\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + } + }, + { + "type": "table", + "id": 41, + "title": "Трассы с ошибками (Tempo, TraceQL)", + "description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 79 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "query": "{ status = error }", + "limit": 20, + "tableType": "traces" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + } + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..ac76d2b --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +# Grafana provisioning: dashboard из файлов (infra/grafana/dashboards/*.json монтируется только для чтения). +apiVersion: 1 + +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..9ccd6d5 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,67 @@ +# Grafana provisioning: источники данных (импортируются при старте, ручной настройки нет). +# Grafana сама ничего не хранит из трёх сигналов — только интерфейс и запросы к хранилищам. +# "$$" — буквальный "$" (иначе Grafana подставит переменную окружения). +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + prometheusType: Prometheus + prometheusVersion: 3.15.1 + timeInterval: 15s + manageAlerts: false # правила алертов в Git (infra/prometheus/rules), Grafana их только показывает + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + # trace_id — structured metadata записи (OTLP): ссылка «лог -> трасса» в Tempo. + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: Открыть трассу + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + # «спан -> логи»: те же trace_id в Loki, окно ±5 минут вокруг спана. + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: false + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__trace.traceId}"' + # «спан -> метрики»: задержка и ошибки сервиса за время спана. + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + queries: + - name: API p95 + query: 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))' + nodeGraph: + enabled: true + search: + hide: false + lokiSearch: + datasourceUid: loki diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..1605eed --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,77 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + # Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка + # при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт, + # сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT). + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..9845653 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,52 @@ +# Loki 3.7.9 — хранилище ЛОГОВ (single binary, файловая система в volume lokidata). +# Логи приходят от Collector по OTLP (/otlp/v1/logs). Ресурсные атрибуты service.name, +# service.instance.id, deployment.environment.name становятся индексными метками (service_name, ...); +# атрибуты записи (event, request_id, trace_id, span_id, severity) — structured metadata, по ним +# фильтруют без индекса: {service_name="messenger-api"} | trace_id="". +# Проверка: loki -config.file=loki.yaml -verify-config +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true # нужно для OTLP: trace_id/span_id и атрибуты записи + retention_period: 168h # 7 дней учебной истории + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_query_lookback: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +pattern_ingester: + enabled: false + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..40541b2 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,129 @@ +# OpenTelemetry Collector (otel/opentelemetry-collector-contrib:0.164.0) — единственный приёмник +# телеметрии приложения. Не хранит данные: принимает OTLP, ограничивает память, пакетирует и +# отправляет в три хранилища. Проверка конфигурации: otelcol-contrib validate --config=collector.yaml +# +# api/worker/dispatcher/janitor --OTLP/HTTP :4318--> receiver otlp +# traces -> memory_limiter -> attributes/scrub -> batch -> otlp_grpc/tempo (Tempo :4317) +# logs -> memory_limiter -> attributes/scrub -> batch -> otlp_http/loki (Loki /otlp) +# metrics -> memory_limiter -> batch -> otlp_http/prometheus (Prometheus /api/v1/otlp) +# +# Порты 4317/4318/8888/13133 слушаются только во внутренней сети telemetry (в compose не публикуются). +# +# Поведение при недоступном хранилище (ограниченная буферизация, RAM не растёт бесконечно): +# - у каждого exporter своя очередь в памяти (sending_queue) фиксированного размера: заполнилась -> +# новые пакеты отбрасываются и считаются (otelcol_exporter_enqueue_failed_*); +# - повтор с экспоненциальной задержкой не дольше max_elapsed_time, потом пакет отбрасывается +# (otelcol_exporter_send_failed_*); +# - memory_limiter: при приближении к лимиту receiver отказывает клиентам (SDK получает ошибку и +# сам отбрасывает данные в своей ограниченной очереди), процесс не уходит в OOM; +# - остальные pipeline работают независимо: упал Tempo — логи и метрики продолжают доходить. +# Очередь только в памяти: при рестарте Collector неотправленное теряется (осознанный компромисс, +# file_storage не нужен для учебного стенда). + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + # Первым в каждом pipeline: проверка heap Go раз в секунду. Мягкий предел = limit - spike = 160 MiB + # (receiver начинает отказывать), жёсткий 200 MiB (принудительный GC). Лимит контейнера 384 MiB: + # RSS включает ещё runtime Go и отображённый в память бинарник contrib (~150 MiB, вытесняемые страницы). + memory_limiter: + check_interval: 1s + limit_mib: 200 + spike_limit_mib: 40 + + # Защита в глубину: приложение не пишет секреты в атрибуты, но если бы инструментирование ошиблось, + # такие ключи не дойдут до хранилищ. Удаляются по шаблону имени ключа (спаны и логи). + attributes/scrub: + actions: + - pattern: (?i).*(password|passwd|authorization|cookie|secret|token|api[_.-]?key|signature).* + action: delete + - key: db.statement.parameters + action: delete + - key: db.query.parameter + action: delete + + batch: + timeout: 2s + send_batch_size: 1024 + send_batch_max_size: 2048 + +exporters: + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true # lab6: внутренняя сеть; TLS/mTLS по trust matrix — в lab7 + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 # пакетов (до 2048 спанов каждый) + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 120s + + otlp_http/loki: + endpoint: http://loki:3100/otlp # -> /otlp/v1/logs + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 120s + + otlp_http/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp # -> /api/v1/otlp/v1/metrics + timeout: 5s + sending_queue: + enabled: true + num_consumers: 1 # один поток: точки одной серии приходят по порядку + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 120s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + # Собственные метрики Collector (принято/отправлено/отброшено, размер очередей, RSS) — + # Prometheus забирает их pull-ом с otel-collector:8888. Так потери телеметрии видны и считаются. + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_grpc/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_http/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/prometheus] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..f0321b7 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +# Prometheus 3.15.1 — хранилище МЕТРИК. Принимает метрики приложения push-ом по OTLP от Collector +# (флаг --web.enable-otlp-receiver, путь /api/v1/otlp/v1/metrics) и сам забирает (pull) служебные +# метрики Collector, Loki и Tempo. Вычисляет recording rules (SLI) и alert rules; состояние алертов +# (pending/firing/resolved) видно на /alerts и через /api/v1/alerts — внешняя доставка не нужна. +# Проверка: promtool check config prometheus.yml +global: + scrape_interval: 15s + scrape_timeout: 5s + evaluation_interval: 15s + +# Как имена OTLP переводятся в имена Prometheus (фиксируем явно, чтобы dashboard не сломался при +# смене значения по умолчанию): точки -> подчёркивания, суффиксы единиц и _total. +# http.server.request.duration (s, histogram) -> http_server_request_duration_seconds_{bucket,sum,count} +# атрибут http.route -> метка http_route; service.name -> job; service.instance.id -> instance. +otlp: + translation_strategy: UnderscoreEscapingWithSuffixes + +storage: + tsdb: + # Collector после недоступности Prometheus дошлёт пакеты из очереди с прошлыми временными метками. + out_of_order_time_window: 10m + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + # Собственные метрики Collector: принято/отправлено/отброшено по сигналам, очереди, RSS. + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] diff --git a/infra/prometheus/rules/alerts.yml b/infra/prometheus/rules/alerts.yml new file mode 100644 index 0000000..1606ac3 --- /dev/null +++ b/infra/prometheus/rules/alerts.yml @@ -0,0 +1,113 @@ +# Alert rules (Prometheus). Внешней доставки нет: состояние inactive/pending/firing видно на +# http://127.0.0.1:9090/alerts, через GET /api/v1/alerts и в Grafana (Alerting -> data source rules); +# метрика ALERTS{alertstate="firing"} хранится в TSDB, поэтому видно и когда алерт стал resolved. +# Каждое правило: условие, окно rate, порог, for, описание и ссылка на локальный runbook. +# Проверка: promtool check rules alerts.yml; promtool test rules ../tests/alerts_test.yml +groups: + - name: messenger-alerts + interval: 15s + rules: + # (1) Высокая доля ошибок API. Окно 1m, порог 5% (в 5 раз выше бюджета SLO-1 = 1%), for 1m, + # и не меньше 0,2 запроса/с, чтобы единичный 503 при пустом трафике не будил дежурного. + - alert: MessengerHighErrorRate + expr: | + ( + (sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) or on() vector(0)) + / sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) + ) > 0.05 + and on() + sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) > 0.2 + for: 1m + labels: + severity: page + service: messenger-api + slo: availability + annotations: + summary: "API: {{ $value | humanizePercentage }} бизнес-запросов завершаются 5xx (порог 5%)" + description: >- + Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше + минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка + кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу. + runbook_url: docs/runbooks.md#messengerhigherrorrate + + # (2) Растущий lag очереди: возраст старейшего ГОТОВОГО к обработке задания (available_at прошёл, + # статус queued) больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно, + # так что правило ловит именно рост, а не единичный всплеск глубины очереди. + - alert: MessengerQueueLagGrowing + expr: max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"}) > 30 + for: 1m + labels: + severity: page + service: messenger-worker + slo: time_to_ready + annotations: + summary: "Очередь attachments.process.v1: старейшее задание ждёт {{ $value | humanizeDuration }}" + description: >- + Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с + дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди, + занятость worker (busy/capacity) и ошибки worker в логах. + runbook_url: docs/runbooks.md#messengerqueuelaggrowing + + # (3) Ни одного consumer у очереди: worker остановлен/упал. Дополняет (2): срабатывает даже без нагрузки. + - alert: MessengerWorkerNoConsumers + expr: max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}) == 0 + for: 1m + labels: + severity: warning + service: messenger-worker + annotations: + summary: "У очереди attachments.process.v1 нет consumers" + description: "RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру." + runbook_url: docs/runbooks.md#messengerworkernoconsumers + + # (4) Насыщение пула соединений API: запросы ждут соединение дольше минуты. + - alert: MessengerDbPoolSaturated + expr: max by (instance) (db_client_connection_pending_requests{job="messenger-api"}) > 0 + for: 1m + labels: + severity: warning + service: messenger-api + annotations: + summary: "Пул PostgreSQL реплики {{ $labels.instance }}: {{ $value }} запросов ждут соединение" + description: "Все DB_POOL_MAX соединений заняты. Причина обычно — медленные запросы или блокировки в PostgreSQL, а не нехватка соединений." + runbook_url: docs/runbooks.md#messengerdbpoolsaturated + + # (5) Collector не может доставить телеметрию в Tempo/Loki/Prometheus: либо пакеты уже теряются + # (send_failed — истёк повтор 120 с у пакета, enqueue_failed — очередь полна), либо очередь exporter + # дольше минуты заполнена больше чем на 5% (хранилище не принимает, пакеты копятся). Во время пакет + # ждёт в очереди его окно повторов не идёт, поэтому без второго условия алерт сработал бы только + # после первых потерь, то есть через 2+ минуты недоступности. + - alert: TelemetryExportFailing + expr: | + sum by (exporter) ( + rate(otelcol_exporter_send_failed_spans[2m]) + or rate(otelcol_exporter_send_failed_log_records[2m]) + or rate(otelcol_exporter_send_failed_metric_points[2m]) + or rate(otelcol_exporter_enqueue_failed_spans[2m]) + or rate(otelcol_exporter_enqueue_failed_log_records[2m]) + or rate(otelcol_exporter_enqueue_failed_metric_points[2m]) + ) > 0 + or + max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.05 + for: 1m + labels: + severity: warning + service: otel-collector + annotations: + summary: "Collector не доставляет телеметрию: exporter {{ $labels.exporter }}" + description: "Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*." + runbook_url: docs/runbooks.md#telemetryexportfailing + + # (6) Collector недоступен: Prometheus не может забрать его служебные метрики. Метрики приложения + # в этот период не доходят (их везёт Collector), поэтому алерт смотрит на up самого Collector. + - alert: TelemetryCollectorDown + expr: up{job="otel-collector"} == 0 + for: 1m + labels: + severity: warning + service: otel-collector + annotations: + summary: "OTel Collector недоступен" + description: "Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском." + runbook_url: docs/runbooks.md#telemetrycollectordown + diff --git a/infra/prometheus/rules/recording.yml b/infra/prometheus/rules/recording.yml new file mode 100644 index 0000000..46ad07b --- /dev/null +++ b/infra/prometheus/rules/recording.yml @@ -0,0 +1,50 @@ +# Recording rules: SLI и агрегаты для dashboard. Проверка: promtool check rules recording.yml +# +# Числитель/знаменатель ошибок API (contracts/observability.md): +# знаменатель — все бизнес-запросы API (job="messenger-api", маршрут НЕ /health/*), любой код; +# числитель — из них ответы 5xx (отказ сервера: БД/Redis/S3 недоступны, внутренняя ошибка). +# 4xx (неверный пароль, 404 чужого чата, 409) — поведение клиента, в ошибки SLO не входят. +# Percentile — только из buckets гистограммы, просуммированных по всем репликам ДО histogram_quantile +# (sum by (le)); p95 отдельных реплик не усредняются. +groups: + - name: messenger-api-sli + interval: 15s + rules: + - record: messenger:http_requests:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) + - record: messenger:http_errors_5xx:rate1m + expr: | + sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) + or on() vector(0) + - record: messenger:http_error_ratio:rate1m + expr: messenger:http_errors_5xx:rate1m / messenger:http_requests:rate1m + - record: messenger:http_request_duration_seconds:p50_5m + expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:http_request_duration_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:http_request_duration_seconds:p99_5m + expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + + # SLO (учебное скользящее окно 1 ч; в эксплуатации то же выражение с [28d]): + # SLO-1 доступность: >= 99% бизнес-запросов без 5xx; + # SLO-2 задержка: >= 95% POST /api/v1/chats/{chat_id}/messages быстрее 250 мс; + # SLO-3 фоновая обработка: >= 95% вложений ready не позже 30 с после приёма (202). + - name: messenger-slo + interval: 30s + rules: + - record: messenger:slo_availability:ratio_1h + expr: | + 1 - ( + (sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1h])) or on() vector(0)) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1h])) + ) + - record: messenger:slo_availability:error_budget_remaining_1h + expr: 1 - (1 - messenger:slo_availability:ratio_1h) / (1 - 0.99) + - record: messenger:slo_latency_messages:ratio_1h + expr: | + sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx", le="0.25"}[1h])) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}[1h])) + - record: messenger:slo_time_to_ready:ratio_1h + expr: | + sum(increase(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) + / sum(increase(messenger_attachment_time_to_ready_seconds_count[1h])) diff --git a/infra/prometheus/tests/alerts_test.yml b/infra/prometheus/tests/alerts_test.yml new file mode 100644 index 0000000..dc660cf --- /dev/null +++ b/infra/prometheus/tests/alerts_test.yml @@ -0,0 +1,121 @@ +# Unit-тесты alert rules на синтетических рядах (без стенда): +# promtool test rules infra/prometheus/tests/alerts_test.yml +# docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml +# Проверяют порог, окно и for: правило не срабатывает раньше времени и при пустом трафике. +rule_files: + - ../rules/alerts.yml +evaluation_interval: 15s + +tests: + # MessengerHighErrorRate: 2 rps успешных, с 3-й минуты +1 rps 5xx (доля 33%). + - interval: 15s + input_series: + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}' + values: '0+30x60' + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="5xx"}' + values: '0x12 0+15x48' + # health-проверки с ошибками не считаются (знаменатель и числитель без /health) + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/health/ready", http_response_status_class="5xx"}' + values: '0+100x60' + alert_rule_test: + - eval_time: 3m + alertname: MessengerHighErrorRate + exp_alerts: [] + - eval_time: 3m45s # условие выполняется, но for: 1m ещё не прошёл + alertname: MessengerHighErrorRate + exp_alerts: [] + - eval_time: 5m + alertname: MessengerHighErrorRate + exp_alerts: + - exp_labels: {severity: page, service: messenger-api, slo: availability} + exp_annotations: + summary: 'API: 33.33% бизнес-запросов завершаются 5xx (порог 5%)' + description: Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу. + runbook_url: docs/runbooks.md#messengerhigherrorrate + + # Пустой трафик: единичные 5xx при 0,05 rps не будят дежурного (порог 0,2 запроса/с). + - interval: 15s + input_series: + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/users/me", http_response_status_class="5xx"}' + values: '0+1x40' + alert_rule_test: + - eval_time: 9m + alertname: MessengerHighErrorRate + exp_alerts: [] + + # MessengerQueueLagGrowing: возраст старейшего задания растёт со 2-й минуты, затем worker вернулся. + - interval: 15s + input_series: + - series: 'messenger_queue_oldest_age_seconds{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}' + values: '0x8 0+15x20 0x20' + - series: 'messenger_queue_consumers{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}' + values: '2x8 0x20 2x20' + alert_rule_test: + - eval_time: 2m30s # возраст 30 с — ещё не больше порога + alertname: MessengerQueueLagGrowing + exp_alerts: [] + - eval_time: 4m + alertname: MessengerQueueLagGrowing + exp_alerts: + - exp_labels: {severity: page, service: messenger-worker, slo: time_to_ready} + exp_annotations: + summary: 'Очередь attachments.process.v1: старейшее задание ждёт 1m 45s' + description: 'Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди, занятость worker (busy/capacity) и ошибки worker в логах.' + runbook_url: docs/runbooks.md#messengerqueuelaggrowing + - eval_time: 4m + alertname: MessengerWorkerNoConsumers + exp_alerts: + - exp_labels: {severity: warning, service: messenger-worker} + exp_annotations: + summary: У очереди attachments.process.v1 нет consumers + description: 'RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру.' + runbook_url: docs/runbooks.md#messengerworkernoconsumers + - eval_time: 9m # worker вернулся, очередь разобрана -> resolved + alertname: MessengerQueueLagGrowing + exp_alerts: [] + + # TelemetryExportFailing и TelemetryCollectorDown + - interval: 15s + input_series: + - series: 'otelcol_exporter_send_failed_spans{exporter="otlp_grpc/tempo"}' + values: '0x8 0+50x20' + - series: 'up{job="otel-collector"}' + values: '1x8 0x20' + alert_rule_test: + - eval_time: 5m + alertname: TelemetryExportFailing + exp_alerts: + - exp_labels: {severity: warning, service: otel-collector, exporter: otlp_grpc/tempo} + exp_annotations: + summary: 'Collector не доставляет телеметрию: exporter otlp_grpc/tempo' + description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.' + runbook_url: docs/runbooks.md#telemetryexportfailing + - eval_time: 5m + alertname: TelemetryCollectorDown + exp_alerts: + - exp_labels: {severity: warning, service: otel-collector, job: otel-collector} + exp_annotations: + summary: OTel Collector недоступен + description: Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском. + runbook_url: docs/runbooks.md#telemetrycollectordown + + # TelemetryExportFailing по заполнению очереди: Loki недоступен, потерь ещё нет, но очередь растёт. + - interval: 15s + input_series: + - series: 'otelcol_exporter_queue_size{exporter="otlp_http/loki"}' + values: '0x8 0+2x30' + - series: 'otelcol_exporter_queue_capacity{exporter="otlp_http/loki"}' + values: '200x38' + alert_rule_test: + - eval_time: 3m # 6 пакетов из 200 (3%) — ниже порога 5% + alertname: TelemetryExportFailing + exp_alerts: [] + - eval_time: 5m + alertname: TelemetryExportFailing + exp_alerts: + - exp_labels: {severity: warning, service: otel-collector, exporter: otlp_http/loki} + exp_annotations: + summary: 'Collector не доставляет телеметрию: exporter otlp_http/loki' + description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.' + runbook_url: docs/runbooks.md#telemetryexportfailing + diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 0000000..f4f41e0 --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,22 @@ +# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json +# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет: +# при импорте определений на старте пользователь по умолчанию не создаётся. +definitions.import_backend = local_filesystem +definitions.local.path = /run/secrets/rabbitmq_definitions +definitions.skip_if_unchanged = true + +listeners.tcp.default = 5672 +loopback_users = none + +# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет), +# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста). +vm_memory_high_watermark.absolute = 320MB +disk_free_limit.absolute = 256MB + +# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение +# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3. +consumer_timeout = 300000 + +log.console = true +log.console.level = info +log.file = false diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 0000000..33c8c08 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..aeebf40 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,39 @@ +# Tempo 3.1.1 — хранилище ТРАСС (single binary: distributor -> live-store -> блоки в volume tempodata, +# без Kafka). Трассы приходят от Collector по OTLP gRPC :4317 (только внутренняя сеть telemetry), +# запросы Grafana и тестов — HTTP :3200 (/api/traces/, /api/search, TraceQL). +# Проверка: tempo -config.file=tempo.yaml -config.verify=true +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + grpc_listen_port: 9095 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +# Хранение 7 дней (как у Loki); удаление старых блоков делают backend-scheduler/worker. +backend_scheduler: + local_work_path: /var/tempo/scheduler + provider: + compaction: + compaction: + block_retention: 168h +backend_worker: + compaction: + block_retention: 168h + +usage_report: + reporting_enabled: false diff --git a/janitor.py b/janitor.py new file mode 100644 index 0000000..904a2fb --- /dev/null +++ b/janitor.py @@ -0,0 +1,322 @@ +"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor). + + python janitor.py --once [--dry-run] [--grace-seconds N] + +Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock): + +1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен + в очередь. Удалить объект staging (идемпотентно), затем строку. +2. Staging после безопасной фиксации: + - ready: исходник уже опубликован в attachments//v/original -> удалить staging-объект + (обычно это делает worker сразу после ack; здесь — если он не успел); + - failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется. + queued/processing не трогаются никогда: это текущая работа. +3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются. +4. Обход bucket по всем prefix (объекты старше grace): + - staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key; + - attachments//...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен: + вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention); + оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing + и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их + теми же ключами, и удаление устаревшим проходом сломало бы новый результат. + Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения: + replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним. + Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных. + +Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому +janitor не удаляет то, что ещё пишется. +""" +import argparse +import json +import logging +import re +import signal +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-janitor") + +import pipeline # noqa: E402 +import storage # noqa: E402 + +log = logging.getLogger("messenger.janitor") + +JANITOR_LOCK = 20260404 +BATCH = 100 +STOP = threading.Event() +ACTOR = "janitor" +FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/") + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def idle(seconds: float) -> bool: + """Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки.""" + deadline = time.monotonic() + seconds + while (left := deadline - time.monotonic()) > 0: + touch_alive() + if STOP.wait(min(left, 10)): + return True + return False + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: + """Шаг 1. Возвращает (найдено, удалено).""" + found = deleted = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, source_key FROM attachments + WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + FOR UPDATE SKIP LOCKED""", + (grace, BATCH), + ).fetchall() + found += len(rows) + if dry_run or not rows: + return found, deleted + for row in rows: + storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) + deleted += 1 + log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) + if len(rows) < BATCH: + return found, deleted + return found, deleted + + +def cleanup_staging(conn, dry_run: bool) -> dict: + """Шаг 2. Исходники ready (сразу) и failed (после retention).""" + report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0} + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a + LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.source_key IS NOT NULL + AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds') + OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s))) + ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""", + (settings.STAGING_RETENTION_SECONDS, BATCH), + ).fetchall() + if dry_run or not rows: + if dry_run: + report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows) + report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows) + break + for row in rows: + storage.delete(row["source_key"]) + conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],)) + if row["state"] == "ready": + report["staging_ready_deleted"] += 1 + else: + report["staging_failed_expired"] += 1 + if row["job_id"] is not None: + pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR, + "replay no longer possible") + if len(rows) < BATCH: + break + report["staging_kept_for_replay"] = conn.execute( + "SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"] + return report + + +def cleanup_outbox(conn, dry_run: bool) -> int: + if dry_run: + return conn.execute( + """SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"] + return conn.execute( + """DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).rowcount + + +def classify_final(conn, keys: list[str]) -> set[str]: + """Какие ключи attachments//... не принадлежат ни одному живому вложению (кандидаты в сироты).""" + ids = {} + for key in keys: + match = FINAL_KEY_RE.match(key) + if match: + try: + ids[key] = UUID(match.group(1)) + except ValueError: + pass + unique = list(set(ids.values())) + rows = {r["id"]: r for r in conn.execute( + """SELECT a.id, a.state, a.object_key, a.source_key, + ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()} + orphans = set() + for key in keys: + attachment_id = ids.get(key) + row = rows.get(attachment_id) if attachment_id else None + if row is None: + orphans.add(key) + elif row["state"] == "failed" and row["source_key"] is None: + # failed без исходника — конечное состояние: replay отказывает («source file is no longer in + # staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи. + orphans.add(key) + elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]: + orphans.add(key) + return orphans + + +def sweep_objects(conn, grace: int, dry_run: bool) -> dict: + """Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" + started = datetime.now(timezone.utc) + cutoff = started - timedelta(seconds=grace) + scanned = orphans = orphans_deleted = 0 + listed = set() + + def orphan_keys(prefix, items): + keys = [key for key, _ in items] + if prefix == storage.FINAL_PREFIX: + return classify_final(conn, keys) + known = {r["source_key"] for r in conn.execute( + "SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()} + return {key for key in keys if key not in known} + + def delete_if_orphan(prefix, key) -> bool: + """Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker.""" + if prefix != storage.FINAL_PREFIX: + # staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется, + # поэтому «нет строки сейчас» означает «не будет никогда». + if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None: + return False + storage.delete(key) + return True + match = FINAL_KEY_RE.match(key) + try: + attachment_id = UUID(match.group(1)) if match else None + except ValueError: + attachment_id = None + with conn.transaction(): + if attachment_id is not None: + # FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но + # конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT. + conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,)) + if key not in classify_final(conn, [key]): + return False + storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе + return True + + def check(prefix, items): + nonlocal orphans, orphans_deleted + candidates = orphan_keys(prefix, items) + for key, modified in items: + if key not in candidates or modified >= cutoff: + continue + orphans += 1 + if not dry_run and delete_if_orphan(prefix, key): + orphans_deleted += 1 + log.info("deleted orphan object %s", key) + + for prefix in storage.ALL_PREFIXES: + page = [] + for key, modified, _size in storage.list_objects(prefix): + scanned += 1 + listed.add(key) + page.append((key, modified)) + if len(page) >= 1000: + check(prefix, page) + page = [] + if page: + check(prefix, page) + + missing = [] + for r in conn.execute( + """SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall(): + if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]): + missing.append(str(r["id"])) + if missing: + log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s", + ", ".join(missing[:20])) + return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, + "missing_objects": len(missing)} + + +def run_once(grace: int, dry_run: bool) -> dict | None: + # Один проход = одна трасса (раз в JANITOR_INTERVAL_SECONDS): запросы к БД и S3 — её дети. + with telemetry.internal_span("janitor pass", {"messenger.janitor.dry_run": dry_run}) as span: + report = _run_once(grace, dry_run) + if report: + span.set_attributes({f"messenger.janitor.{k}": v for k, v in report.items() + if isinstance(v, (int, float, bool))}) + return report + + +def _run_once(grace: int, dry_run: bool) -> dict | None: + started = time.monotonic() + with connect() as conn: + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: + log.info("another janitor is running, skip") + return None + try: + found, deleted = reclaim_stale_intents(conn, grace, dry_run) + report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, + "stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run), + "outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)} + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) + report["seconds"] = round(time.monotonic() - started, 3) + return report + + +def main(): + parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects") + parser.add_argument("--once", action="store_true", help="one pass and exit") + parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") + parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) + args = parser.parse_args() + if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: + parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " + "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") + + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + while True: + touch_alive() + try: + report = run_once(args.grace_seconds, args.dry_run) + if report is not None: + print(json.dumps(report), flush=True) + log.info("janitor pass done", extra={"event": "janitor.pass", **{ + k: v for k, v in report.items() if isinstance(v, (int, float, bool))}}) + failed = False + except (psycopg.Error, storage.StorageUnavailable) as exc: + log.warning("pass failed, will retry: %s", type(exc).__name__, + extra={"event": "janitor.pass_failed", "error.type": type(exc).__name__}) + failed = True + if args.once: + telemetry.shutdown() + raise SystemExit(1 if failed else 0) + if idle(settings.JANITOR_INTERVAL_SECONDS): + telemetry.shutdown() + return + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 0000000..4f93308 --- /dev/null +++ b/main.py @@ -0,0 +1,666 @@ +import base64 +import binascii +import contextlib +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path +from typing import Literal +from urllib.parse import quote +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from opentelemetry import trace +from opentelemetry.trace import SpanKind +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.routing import Match + +import settings +import telemetry + +# lab6: телеметрия настраивается до первого соединения с БД (pool.open в lifespan). +telemetry.setup("messenger-api") + +import attachments # noqa: E402 +import auth # noqa: E402 +import db # noqa: E402 +import storage # noqa: E402 +import uploads # noqa: E402 +from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable # noqa: E402 + +log = logging.getLogger("messenger") + +# HTTP-метрики (contracts/observability.md). Метки: метод, ШАБЛОН маршрута, код и класс статуса — +# никаких реальных UUID, raw URL, request_id. Длительность — до отправки заголовков ответа +# (для скачивания файла тело потом стримится из S3 и в длительность не входит). +HTTP_DURATION = telemetry.meter.create_histogram( + "http.server.request.duration", unit="s", description="Duration of HTTP server requests") +HTTP_ACTIVE = telemetry.meter.create_up_down_counter( + "http.server.active_requests", unit="{request}", description="HTTP requests being processed now") +KNOWN_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + # Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3. + Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, " + "processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, + settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION, + extra={"event": "service.started"}) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + telemetry.shutdown() + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + # С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("attachment_ids must be unique") + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text cannot be empty without attachments") + return self + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] + + +class Variant(BaseModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +def route_template(request: Request) -> str: + """Шаблон маршрута (/api/v1/chats/{chat_id}/messages) — метка метрик и имя спана вместо raw URL.""" + for route in app.router.routes: + match, _ = route.matches(request.scope) + if match != Match.NONE: + return getattr(route, "path", "unmatched") + return "unmatched" + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + method = request.method if request.method in KNOWN_METHODS else "_OTHER" + route = route_template(request) + # Проверки здоровья (HAProxy раз в секунду, Docker) — только в метриках, без трасс: иначе это + # большая часть всех трасс. Входящий traceparent клиента продолжается (W3C Trace Context). + traced = not path.startswith("/health/") + span_scope = (telemetry.tracer.start_as_current_span( + f"{method} {route}", context=telemetry.extract(request.headers), kind=SpanKind.SERVER, + attributes={"http.request.method": method, "http.route": route, "url.scheme": "http", + "request.id": request_id}, + record_exception=False, set_status_on_exception=False) + if traced else contextlib.nullcontext(trace.INVALID_SPAN)) + HTTP_ACTIVE.add(1, {"http.request.method": method}) + status = 500 + try: + with span_scope as span: + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception as exc: + log.exception("Unhandled error", extra={"event": "http.unhandled_error", "request_id": request_id}) + telemetry.mark_error(span, type(exc).__name__) + response = error_response(request, "internal_error", "Internal server error", 500) + + status = response.status_code + span.set_attribute("http.response.status_code", status) + if status >= 500: + telemetry.mark_error(span, str(status)) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + trace_id = telemetry.trace_id_hex() if traced else None + if trace_id: + response.headers["X-Trace-Id"] = trace_id # расширение: найти трассу по ответу + elapsed = time.perf_counter() - started + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if not traced and status == 200 else logging.INFO + log.log(level, "%s %s -> %s (%.1f ms)", method, route, status, elapsed * 1000, + extra={"event": "http.request", "http.request.method": method, "http.route": route, + "http.response.status_code": status, "duration_ms": round(elapsed * 1000, 1), + "request_id": request_id}) + finally: + HTTP_ACTIVE.add(-1, {"http.request.method": method}) + elapsed = time.perf_counter() - started + HTTP_DURATION.record(elapsed, {"http.request.method": method, "http.route": route, + "http.response.status_code": status, + "http.response.status_class": f"{status // 100}xx"}) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + if cursor is None: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют. + + Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []} + равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними. + """ + body = data.model_dump(mode="json") + if not body.get("attachment_ids"): + body.pop("attachment_ids", None) + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=Attachment, + status_code=202, +) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str | None = Query(default=None), +): + """Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение + для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker. + + Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415) + -> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413), + проверка содержимого (415/413) и запись. + """ + user_id = current_user_id(request) + try: + await run_in_threadpool(db.check_member, chat_id, user_id) + name = uploads.validate_filename(filename) + declared = uploads.declared_type(request.headers.get("content-type")) + uploads.declared_length(request.headers.get("content-length")) + except ApiError: + await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке + raise + upload = await uploads.receive(request) + try: + image = await run_in_threadpool(inspect_traced, upload, declared) + row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image) + finally: + upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3) + return attachments.public(row) + + +def inspect_traced(upload, declared): + """Проверка содержимого — CPU в обработчике (не БД): в трассе видно отдельным спаном.""" + with telemetry.internal_span("inspect image", {"http.request.body.size": upload.size}): + return uploads.inspect(upload, declared) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public(db.get_attachment(attachment_id, current_user_id(request))) + + +VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"} + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + request: Request, + attachment_id: UUID, + variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"), +): + """Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3. + + queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404. + """ + row = db.get_attachment(attachment_id, current_user_id(request)) + if row["state"] != "ready": + raise conflict("Attachment is not ready") + if variant == "original": + key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"] + name = row["filename"] + else: + found = [v for v in row["variants"] if v["name"] == variant] + if not found: + raise not_found("Variant not found") + v = found[0] + key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"] + stem = row["filename"].rsplit(".", 1)[0] or "image" + name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}" + try: + stream = storage.open_read(key) + except storage.ObjectMissing: + log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"], + extra={"event": "attachment.object_missing", "attachment.id": str(row["id"])}) + raise unavailable("File is temporarily unavailable") from None + except storage.StorageUnavailable: + raise unavailable("File storage is temporarily unavailable") from None + headers = { + "Content-Length": str(size), + "Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + sha256 + '"', + } + return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + log_config=None, # логи uvicorn идут в общий JSON/OTLP-обработчик (telemetry.py) + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..e95f414 --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..8b981d0 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 0000000..73f0c84 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..5770e1d --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..35697dd --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,48 @@ +-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением. +-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать. + +-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата. +ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + -- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит. + object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'), + -- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна; + -- ready: объект в S3 сохранён, можно читать и прикреплять. + state text NOT NULL CHECK (state IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (width::bigint * height <= 40000000), + UNIQUE (id, chat_id), + -- владелец на момент загрузки был участником именно этого чата + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); + +-- janitor ищет старые незавершённые загрузки +CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + -- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id); + +-- С lab4 текст может быть пустым, если есть вложение (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/migrations/005_jobs.sql b/migrations/005_jobs.sql new file mode 100644 index 0000000..0751798 --- /dev/null +++ b/migrations/005_jobs.sql @@ -0,0 +1,133 @@ +-- lab5: асинхронная обработка вложений. Задания, outbox, варианты, журнал событий. +-- +-- Состояния attachments.state: +-- uploading — строка-намерение до записи в staging (как в lab4), клиенту не видна; +-- queued / processing / ready / failed — публичные состояния из контракта. +-- source_key — исходник в staging (staging/; у старых вложений lab4 — originals/); +-- object_key — опубликованный оригинал attachments//v/original, есть только у ready. + +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_check; +ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL; +ALTER TABLE attachments ADD COLUMN source_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer; + +-- Вложения lab4 были приняты синхронно и не имеют вариантов: переносим их в staging-модель и ставим +-- в очередь (ниже), worker опубликует оригинал в конечный prefix и сделает варианты. +UPDATE attachments SET source_key = object_key, object_key = NULL; +UPDATE attachments SET state = 'queued', updated_at = now() WHERE state = 'ready'; + +ALTER TABLE attachments ADD CONSTRAINT attachments_state_check + CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD CONSTRAINT attachments_source_key_check + CHECK (source_key ~ '^(staging|originals)/[a-f0-9]{32}$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_check + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/original$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code_check + CHECK (error_code ~ '^[a-z_]{1,64}$'); +-- ready <=> опубликованный оригинал; failed <=> код ошибки; до ready исходник обязан быть в staging. +ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object + CHECK ((state = 'ready') = (object_key IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_failed_has_code + CHECK ((state = 'failed') = (error_code IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_pending_has_source + CHECK (state NOT IN ('uploading', 'queued', 'processing') OR source_key IS NOT NULL); + +-- janitor: исходники ready/failed, которые пора удалить из staging +CREATE INDEX attachments_staging ON attachments (updated_at) + WHERE source_key IS NOT NULL AND state IN ('ready', 'failed'); + +-- Варианты. Первичный ключ = ключ эффекта (attachment_id, pipeline_version, variant): +-- повтор job не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/[a-z]+\.(jpg|png|webp)$'), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Задание: одна логическая работа на (вложение, версия pipeline). job_id не меняется при повторах. +-- attempt — управляемая попытка (claim worker'ом); lease_* — аренда processing с fencing-токеном. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts integer NOT NULL CHECK (max_attempts >= 1), + available_at timestamptz NOT NULL DEFAULT now(), + lease_token uuid, + lease_owner text, + lease_until timestamptz, + error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'), + last_error text CHECK (last_error ~ '^[a-z_]{1,64}$'), + replays integer NOT NULL DEFAULT 0, + traceparent text CHECK (traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + UNIQUE (attachment_id, pipeline_version), + CHECK (attempt <= max_attempts), + CHECK ((status = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)), + CHECK ((status = 'failed') = (error_code IS NOT NULL)) +); +CREATE INDEX attachment_jobs_leases ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_queued ON attachment_jobs (available_at) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_failed ON attachment_jobs (finished_at) WHERE status = 'failed'; + +-- Журнал (audit): claim, повторы, истечение аренды, дубли, replay. Только коды, без секретов и путей. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + event text NOT NULL CHECK (event ~ '^[a-z_]{1,64}$'), + detail text CHECK (char_length(detail) <= 200), + actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX job_events_by_job ON job_events (job_id, id); + +-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и задание. +-- dispatcher публикует строки с available_at <= now() и отмечает published_at после publisher confirm. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0 +); +CREATE INDEX outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_by_job ON outbox (job_id); + +-- Задания для вложений, принятых в lab4 (миграция = их producer). JOB_MAX_ATTEMPTS по умолчанию 5. +INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent) +SELECT gen_random_uuid(), id, 1, 'queued', 1, 5, + '00-' || md5(random()::text || id::text) || '-' || substr(md5(id::text), 1, 16) || '-01' +FROM attachments WHERE state = 'queued'; + +INSERT INTO job_events (job_id, attempt, event, detail, actor) +SELECT job_id, 1, 'created', 'lab4_backfill', 'migration 005' FROM attachment_jobs; + +INSERT INTO outbox (job_id, attempt, payload) +SELECT job_id, 1, jsonb_build_object( + 'schema_version', 1, + 'job_id', job_id, + 'type', 'attachment.process.v1', + 'attachment_id', attachment_id, + 'pipeline_version', pipeline_version, + 'attempt', 1, + 'created_at', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'traceparent', traceparent) +FROM attachment_jobs; diff --git a/ops.py b/ops.py new file mode 100644 index 0000000..d19a63d --- /dev/null +++ b/ops.py @@ -0,0 +1,201 @@ +"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher). + + docker compose exec dispatcher python ops.py status + docker compose exec dispatcher python ops.py failed [--limit N] + docker compose exec dispatcher python ops.py events + docker compose exec dispatcher python ops.py replay [--reason TEXT] + docker compose exec dispatcher python ops.py dlq [--limit N] + docker compose exec dispatcher python ops.py dlq-replay [--limit N] + +Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code, +число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя +обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit. + +replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready, +queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка +идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant). + +dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины): +failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу +ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения +остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3. +""" +import argparse +import json +import sys +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-ops") + +import broker # noqa: E402 +import pipeline # noqa: E402 + +ACTOR = "ops@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options="-c TimeZone=UTC") + + +def text(value): + if isinstance(value, (UUID,)): + return str(value) + if hasattr(value, "isoformat"): + return pipeline.utc_text(value) + return value + + +def rows_json(rows): + return [{key: text(value) for key, value in row.items()} for row in rows] + + +def cmd_status(conn, _args): + jobs = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()} + outbox = conn.execute( + """SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending, + count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due, + coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures, + extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL + AND available_at <= now())) AS oldest_due_s + FROM outbox""").fetchone() + leases = conn.execute( + """SELECT count(*) FILTER (WHERE lease_until >= now()) AS active, + count(*) FILTER (WHERE lease_until < now()) AS expired + FROM attachment_jobs WHERE status = 'processing'""").fetchone() + staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"] + report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v) + for k, v in outbox.items()}, + "leases": leases, "attachments_with_staging_source": staging} + try: + connection = broker.connect() + try: + report["broker"] = broker.check_topology(connection.channel()) + finally: + connection.close() + except broker.BrokerError as exc: + report["broker"] = {"error": type(exc).__name__} + return report + + +def cmd_failed(conn, args): + return rows_json(conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, + j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall()) + + +def cmd_events(conn, args): + return rows_json(conn.execute( + "SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id", + (args.job_id,)).fetchall()) + + +def cmd_replay(conn, args): + try: + return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)} + except pipeline.ReplayRefused as exc: + return {"refused": str(exc), "job_id": str(args.job_id)} + + +def dlq_messages(limit, handler): + """Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'.""" + connection = broker.connect() + results = [] + try: + channel = connection.channel() + keep = [] + for _ in range(limit): + method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False) + if method is None: + break + death = (properties.headers or {}).get("x-death") or [{}] + reason = death[0].get("reason") + reason = reason.decode() if isinstance(reason, bytes) else reason + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + message, problem = None, str(exc) + else: + problem = None + verdict, note = handler(message) + results.append({"job_id": message["job_id"] if message else None, "invalid": problem, + "dead_letter_reason": reason, "action": verdict, "note": note}) + if verdict == "ack": + channel.basic_ack(method.delivery_tag) + else: + keep.append(method.delivery_tag) + for tag in keep: # вернуть непрочитанными в DLQ + channel.basic_nack(tag, requeue=True) + finally: + connection.close() + return results + + +def cmd_dlq(_conn, args): + return dlq_messages(args.limit, lambda message: ("keep", "peek")) + + +def cmd_dlq_replay(conn, args): + def handle(message): + if message is None: + return "keep", "invalid message is kept for analysis" + job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s", + (UUID(message["job_id"]),)).fetchone() + if job is None: + return "keep", "unknown job" + if job["status"] == "failed": + try: + pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay") + return "ack", "replayed" + except pipeline.ReplayRefused as exc: + return "keep", str(exc) + if job["status"] == "ready": + return "ack", "already ready, no-op" + # queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново + conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s " + "AND status = 'queued'", (UUID(message["job_id"]),)) + return "ack", f"job is {job['status']}, handled by outbox/lease recovery" + return dlq_messages(args.limit, handle) + + +def main(): + parser = argparse.ArgumentParser(description="Attachment pipeline operator commands") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + p = sub.add_parser("failed") + p.add_argument("--limit", type=int, default=50) + p = sub.add_parser("events") + p.add_argument("job_id", type=UUID) + p = sub.add_parser("replay") + p.add_argument("job_id", type=UUID) + p.add_argument("--reason", default="operator replay") + p = sub.add_parser("dlq") + p.add_argument("--limit", type=int, default=20) + p = sub.add_parser("dlq-replay") + p.add_argument("--limit", type=int, default=100) + args = parser.parse_args() + if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()): + parser.error("--reason: up to 100 printable characters") + commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay, + "dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay} + # Действие оператора — отдельная трасса; replay ссылается (link) на трассу исходной загрузки. + with telemetry.internal_span(f"ops {args.command}", {"messenger.ops.command": args.command}): + with connect_db() as conn: + result = commands[args.command](conn, args) + telemetry.shutdown() + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..8cbe2cb --- /dev/null +++ b/pipeline.py @@ -0,0 +1,502 @@ +"""Задания обработки вложений: формат сообщения, outbox, переходы состояний. + +Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат) +и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции +вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT. + +Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать, +решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение +безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op. + +Переходы (contracts/attachment-states.md): + (нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202 + queued -> processing claim worker, аренда lease_until + fencing-токен lease_token + processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter) + recover_lease аренда истекла (worker умер/завис), попытка засчитана + processing -> ready finish_ready только при совпадении lease_token (fencing) + processing -> failed finish_failed постоянная ошибка или попытки исчерпаны + failed -> queued replay явная операция оператора (ops.py), с записью в журнал +""" +import json +import random +import re +import secrets +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from opentelemetry import trace +from opentelemetry.trace import SpanKind +from psycopg.types.json import Jsonb + +import settings +import telemetry + +SCHEMA_VERSION = 1 +JOB_TYPE = "attachment.process.v1" +EXCHANGE = "attachments" +ROUTING_KEY = "attachment.process.v1" +QUEUE = "attachments.process.v1" +DLX = "attachments.dlx" +DLQ = "attachments.process.v1.dlq" +DLQ_ROUTING_KEY = "attachment.process.v1.dlq" + +MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", + "created_at", "traceparent", "tracestate"} +REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"} +TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +CODE_RE = re.compile(r"^[a-z_]{1,64}$") + +# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES. +PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"} +FINAL_CODES = PERMANENT | {"retry_exhausted"} + + +class InvalidMessage(Exception): + """Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ.""" + + +def utc_text(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def new_traceparent() -> str: + """Запасной traceparent, если телеметрия выключена (OTEL_SDK_DISABLED): поле схемы остаётся заполненным.""" + return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01" + + +def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str | None, tracestate: str | None = None) -> dict: + """Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3. + + lab6: traceparent — контекст РЕАЛЬНОГО producer-спана, создавшего строку outbox (а не случайный ID). + Dispatcher при публикации заменяет его контекстом своего спана publish (тот же trace_id). + """ + message = { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": utc_text(datetime.now(timezone.utc)), + } + if traceparent: + message["traceparent"] = traceparent + if tracestate: + message["tracestate"] = tracestate[:512] + return message + + +def messaging_attributes(job_id: UUID, attachment_id: UUID, attempt: int, operation: str) -> dict: + """Атрибуты messaging-спанов (OTel semantic conventions) + технические ID задания.""" + return {"messaging.system": "rabbitmq", "messaging.operation.type": operation, + "messaging.destination.name": EXCHANGE, "messaging.rabbitmq.destination.routing_key": ROUTING_KEY, + "messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id), + "messenger.attachment.id": str(attachment_id), "messenger.job.attempt": attempt} + + +@contextmanager +def producer_span(job_id: UUID, attachment_id: UUID, attempt: int, parent_traceparent: str | None = None, + links: list | None = None): + """Спан «create attachments» (PRODUCER): строка outbox = сообщение, созданное в этой транзакции. + + Родитель — текущий спан (запрос API, попытка worker), а если его нет (фоновый dispatcher) — + parent_traceparent из строки задания. Возвращает (traceparent, tracestate) для сообщения. + """ + current = trace.get_current_span().get_span_context() + parent = None if current.is_valid else telemetry.context_from(parent_traceparent) + with telemetry.tracer.start_as_current_span( + f"create {EXCHANGE}", context=parent, kind=SpanKind.PRODUCER, links=links, + attributes=messaging_attributes(job_id, attachment_id, attempt, "create")): + carrier = telemetry.inject() + yield carrier.get("traceparent") or new_traceparent(), carrier.get("tracestate") + + +def parse_message(body: bytes) -> dict: + """Строгая проверка по схеме (additionalProperties: false, const, типы, форматы).""" + try: + message = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidMessage("not json") from None + if not isinstance(message, dict): + raise InvalidMessage("not an object") + if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message): + raise InvalidMessage("fields") + if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE: + raise InvalidMessage("schema_version/type") + for name in ("job_id", "attachment_id"): + if not isinstance(message[name], str) or not UUID_RE.match(message[name]): + raise InvalidMessage(name) + for name in ("pipeline_version", "attempt"): + value = message[name] + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidMessage(name) + try: + created = message["created_at"] + if not isinstance(created, str): + raise ValueError + datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError: + raise InvalidMessage("created_at") from None + if "traceparent" in message and (not isinstance(message["traceparent"], str) + or not TRACEPARENT_RE.match(message["traceparent"])): + raise InvalidMessage("traceparent") + if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512): + raise InvalidMessage("tracestate") + return message + + +def backoff_seconds(failed_attempt: int) -> float: + """Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна. + + Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS. + Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу). + """ + ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS, + settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1)) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None: + conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)", + (job_id, attempt, name, detail[:200] if detail else None, actor[:200])) + + +def _insert_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str, tracestate: str | None, delay_seconds: float) -> None: + payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) + conn.execute( + """INSERT INTO outbox (job_id, attempt, payload, available_at) + VALUES (%s, %s, %s, now() + make_interval(secs => %s))""", + (job_id, attempt, Jsonb(payload), delay_seconds), + ) + conn.execute("NOTIFY outbox") + + +def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + parent_traceparent: str | None, delay_seconds: float = 0, links: list | None = None) -> None: + """Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT.""" + with producer_span(job_id, attachment_id, attempt, parent_traceparent, links) as (traceparent, tracestate): + _insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate, + delay_seconds) + + +def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID: + """Новое задание для вложения, уже переведённого в queued (в транзакции API). + + attachment_jobs.traceparent — контекст исходной загрузки (спан create внутри запроса API): к нему + привязываются восстановление аренды и сверка, а replay оператора ссылается на него через link. + """ + job_id = uuid4() + with producer_span(job_id, attachment_id, 1) as (traceparent, tracestate): + conn.execute( + """INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, + traceparent) + VALUES (%s, %s, %s, 'queued', 1, %s, %s)""", + (job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent), + ) + event(conn, job_id, 1, "created", actor) + _insert_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent, tracestate, 0) + return job_id + + +# ---------- worker: захват и результат ---------- + +@dataclass +class Claim: + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + max_attempts: int + lease_token: UUID + source_key: str + content_type: str + size_bytes: int + sha256: str + traceparent: str | None + + +@dataclass +class Skip: + """Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker).""" + reason: str + + +@dataclass +class Reject: + """Сообщение не относится ни к одной job -> DLQ.""" + reason: str + + +def claim(conn, message: dict, owner: str, redelivered: bool): + """queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject.""" + job_id = UUID(message["job_id"]) + with conn.transaction(): + job = conn.execute( + """SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired + FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""", + (job_id,), + ).fetchone() + if job is None: + return Reject("unknown_job") + if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]: + return Reject("job_mismatch") + status = job["status"] + if status in ("ready", "failed"): + event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}") + return Skip(f"already_{status}") + attempt = job["attempt"] + if status == "queued": + if not job["due"]: + return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox + else: # processing + if not job["lease_expired"]: + event(conn, job_id, attempt, "duplicate_ignored", owner, "leased") + return Skip("leased") + # Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана. + event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker") + if attempt >= job["max_attempts"]: + _fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired") + return Skip("exhausted") + attempt += 1 + token = uuid4() + conn.execute( + """UPDATE attachment_jobs + SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, + lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id), + ) + att = conn.execute( + """UPDATE attachments SET state = 'processing', updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing') + RETURNING source_key, content_type, size_bytes, sha256""", + (job["attachment_id"],), + ).fetchone() + if att is None or att["source_key"] is None: + raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат + event(conn, job_id, attempt, "claimed", owner, + f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}") + return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token, + att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"]) + + +def renew_lease(conn, c: Claim) -> bool: + """Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить.""" + row = conn.execute( + """UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""", + (settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def _fenced(conn, c: Claim, owner: str) -> bool: + """Fencing: результат принимается, только если lease_token всё ещё наш.""" + row = conn.execute( + """SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing' + FOR UPDATE""", + (c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> float | None: + """processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT. + + Возвращает время от устойчивого приёма (COMMIT задания в API) до ready в секундах по часам + PostgreSQL (метрика messenger.attachment.time_to_ready) или None, если результат отклонён fencing. + """ + time_to_ready = None + with conn.transaction(): + if not _fenced(conn, c, owner): + stale = True + else: + stale = False + for v in variants: + conn.execute( + """INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""", + (c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"], + v["width"], v["height"], v["sha256"], v["object_key"]), + ) + # Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна). + updated = conn.execute( + """UPDATE attachments + SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now() + WHERE id = %s AND state = 'processing' + AND (pipeline_version IS NULL OR pipeline_version <= %s) + RETURNING id""", + (original_key, c.pipeline_version, c.attachment_id, c.pipeline_version), + ).fetchone() + if updated is None: + raise RuntimeError("attachment changed state during processing") + time_to_ready = conn.execute( + """UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s + RETURNING extract(epoch FROM clock_timestamp() - created_at)::float8 AS seconds""", + (c.job_id,), + ).fetchone()["seconds"] + event(conn, c.job_id, c.attempt, "ready", owner) + if stale: + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost") + return None + return time_to_ready + + +def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None): + conn.execute( + """UPDATE attachment_jobs + SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s""", + (code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id), + ) + conn.execute( + """UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing')""", + (code, attachment_id), + ) + event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}") + + +def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int, + traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None: + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (next_attempt, reason, delay, job_id), + ) + conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'", + (attachment_id,)) + add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay) + event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s") + + +def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str: + """Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'.""" + with conn.transaction(): + if not _fenced(conn, c, owner): + outcome = "stale" + elif permanent: + _fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None) + outcome = "failed" + elif c.attempt >= c.max_attempts: + _fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code) + outcome = "failed" + else: + _requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent, + backoff_seconds(c.attempt), code, owner, "retry_scheduled") + outcome = "retry" + if outcome == "stale": + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code) + return outcome + + +# ---------- dispatcher: аренды и сверка ---------- + +def recover_expired_leases(conn, actor: str, limit: int = 100) -> int: + """processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted).""" + recovered = 0 + with conn.transaction(): + rows = conn.execute( + """SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent + FROM attachment_jobs WHERE status = 'processing' AND lease_until < now() + ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""", + (limit,), + ).fetchall() + for job in rows: + # Спан в трассе исходной загрузки: видно, что попытку вернул dispatcher, а не worker. + with telemetry.tracer.start_as_current_span( + "recover expired lease", context=telemetry.context_from(job["traceparent"]), + attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}): + if job["attempt"] >= job["max_attempts"]: + _fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor, + "lease_expired") + else: + _requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], + job["attempt"] + 1, job["traceparent"], backoff_seconds(job["attempt"]), + "lease_expired", actor, "lease_expired") + recovered += 1 + return recovered + + +def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int: + """Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет. + + Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его + подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет. + """ + with conn.transaction(): + rows = conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent + FROM attachment_jobs j + WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s) + AND j.updated_at < now() - make_interval(secs => %s) + AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND + (o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s))) + ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""", + (older_than, older_than, older_than, limit), + ).fetchall() + for job in rows: + with telemetry.tracer.start_as_current_span( + "republish lost message", context=telemetry.context_from(job["traceparent"]), + attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}): + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], + job["traceparent"]) + conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],)) + event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight") + return len(rows) + + +# ---------- оператор: replay ---------- + +class ReplayRefused(Exception): + pass + + +def replay(conn, job_id: UUID, actor: str, reason: str) -> dict: + """failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются.""" + with conn.transaction(): + job = conn.execute( + """SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""", + (job_id,), + ).fetchone() + if job is None: + raise ReplayRefused("job not found") + if job["status"] != "failed": + # «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается. + raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs") + if job["source_key"] is None: + raise ReplayRefused("source file is no longer in staging (retention expired)") + attempt = job["attempt"] + 1 + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL, + available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now() + WHERE job_id = %s""", + (attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id), + ) + conn.execute( + "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'", + (job["attachment_id"],), + ) + event(conn, job_id, attempt, "replayed", actor, reason or "operator replay") + # Replay — новое решение оператора, а не продолжение исходного запроса: новая трасса со + # ссылкой (link) на исходную загрузку. Вызов из спана ops.py становится его родителем. + add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, None, + links=telemetry.link_to(job["traceparent"])) + return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6bf911f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,49 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +boto3==1.43.116 +botocore==1.43.116 +s3transfer==0.19.3 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 +pillow==12.3.0 +pika==1.3.2 +# lab6: OpenTelemetry SDK (traces/metrics/logs, OTLP/HTTP protobuf) + инструментирование psycopg +opentelemetry-api==1.45.2 +opentelemetry-sdk==1.45.2 +opentelemetry-exporter-otlp-proto-http==1.45.2 +opentelemetry-exporter-otlp-proto-common==1.45.2 +opentelemetry-proto==1.45.2 +opentelemetry-semantic-conventions==0.66b2 +opentelemetry-instrumentation==0.66b2 +opentelemetry-instrumentation-dbapi==0.66b2 +opentelemetry-instrumentation-psycopg==0.66b2 +opentelemetry-exporter-http-transport==0.66b2 +opentelemetry-exporter-otlp-common==0.66b2 +googleapis-common-protos==1.76.0 +protobuf==7.36.2 +requests==2.34.2 +certifi==2026.10.23 +charset-normalizer==3.5.2 +packaging==26.3 +wrapt==2.5.2 diff --git a/s3_init.py b/s3_init.py new file mode 100644 index 0000000..99489bc --- /dev/null +++ b/s3_init.py @@ -0,0 +1,103 @@ +"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init. + +Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает. +Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py): + admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init; + messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor. +Значения ключей не печатаются. +""" +import logging +import sys +import time +import urllib.error +import urllib.request +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +logging.basicConfig(level=logging.WARNING) + + +def client(key_id: str, secret: str): + return boto3.client( + "s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION, + aws_access_key_id=key_id, aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, + read_timeout=10, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + +def code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "?") + + +def anonymous_status(url: str, method: str = "GET") -> int: + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply: + return reply.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = settings.S3_BUCKET + admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"), + settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY) + + deadline = time.monotonic() + 90 + while True: + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if time.monotonic() > deadline: + print(f"s3-init: S3 is not ready: {type(exc).__name__}") + return 1 + time.sleep(1) + + try: + admin.create_bucket(Bucket=bucket) + print(f"s3-init: bucket {bucket} created") + except ClientError as exc: + if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + print(f"s3-init: cannot create bucket: {code(exc)}") + return 1 + print(f"s3-init: bucket {bucket} exists") + + # Ключ API работает со своим bucket ... + probe = "originals/s3-init-probe-" + uuid.uuid4().hex + api.put_object(Bucket=bucket, Key=probe, Body=b"probe") + api.get_object(Bucket=bucket, Key=probe)["Body"].read() + api.list_objects_v2(Bucket=bucket, MaxKeys=1) + api.delete_object(Bucket=bucket, Key=probe) + print("s3-init: api key can put/get/list/delete objects of its bucket") + # ... и не может администрировать хранилище. + intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8] + try: + api.create_bucket(Bucket=intruder) + admin.delete_bucket(Bucket=intruder) + print("s3-init: api key must NOT be able to create buckets") + return 1 + except ClientError as exc: + print(f"s3-init: api key cannot create buckets ({code(exc)})") + + # Приватность: анонимные запросы без подписи. + base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket + statuses = {"list": anonymous_status(base + "?list-type=2"), + "get": anonymous_status(base + "/originals/any"), + "put": anonymous_status(base + "/originals/any", "PUT")} + if any(status not in (401, 403) for status in statuses.values()): + print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}") + return 1 + print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_dashboard.py b/scripts/build_dashboard.py new file mode 100644 index 0000000..71b51c3 --- /dev/null +++ b/scripts/build_dashboard.py @@ -0,0 +1,265 @@ +"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git). + + python scripts/build_dashboard.py + +Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос +против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой. +Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти. +""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "infra/grafana/dashboards/messenger.json" + +PROM = {"type": "prometheus", "uid": "prometheus"} +LOKI = {"type": "loki", "uid": "loki"} +TEMPO = {"type": "tempo", "uid": "tempo"} + +API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO) +QUEUE = 'messaging_destination_name="attachments.process.v1"' + +_panels = [] +_next_id = [1] +_y = [0] + + +def _id(): + _next_id[0] += 1 + return _next_id[0] + + +def row(title): + _panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []}) + _y[0] += 1 + + +class Line: + """Раскладка панелей в строку сетки Grafana (24 колонки).""" + + def __init__(self, height): + self.x, self.h = 0, height + + def place(self, width): + pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]} + self.x += width + return pos + + def end(self): + _y[0] += self.h + + +def prom(expr, legend="", instant=False, fmt=None): + target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "", + "range": not instant, "instant": instant} + if fmt: + target["format"] = fmt + return target + + +def _with_refs(targets): + for n, target in enumerate(targets): + target["refId"] = chr(ord("A") + n) + return targets + + +def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None): + defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never", + "stacking": {"mode": "normal" if stack else "none"}}} + if minimum is not None: + defaults["min"] = minimum + if thresholds: + defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [ + {"color": color, "value": value} for value, color in thresholds]} + defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"} + _panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description, + "datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets), + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"}}}) + + +def stat(line, width, title, description, expr, unit, steps, decimals=None): + defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [ + {"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}} + if decimals is not None: + defaults["decimals"] = decimals + _panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM, + "gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]), + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "colorMode": "background", "graphMode": "none", "textMode": "value"}}) + + +def quantiles(metric, selector, by="le"): + return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))", + f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)] + + +def build(): + row("Обзор и SLO (окно SLO — 1 час)") + line = Line(4) + stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.", + "messenger:http_requests:rate1m", "reqps", [(None, "blue")]) + stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.", + "messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2) + stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).", + "messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")]) + stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.", + f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")]) + stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.", + "messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2) + stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.", + "messenger:slo_availability:error_budget_remaining_1h", "percentunit", + [(None, "red"), (0, "orange"), (0.5, "green")], 0) + stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.", + "messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2) + stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.", + "messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2) + line.end() + line = Line(5) + _panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)", + "description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.", + "datasource": PROM, "gridPos": line.place(24), + "targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]), + "transformations": [{"id": "organize", "options": {"excludeByName": { + "Time": True, "Value": True, "__name__": True}}}], + "fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}}) + line.end() + + row("API: запросы, ошибки, задержка (RED)") + line = Line(8) + timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.", + [prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", + "{{http_request_method}} {{http_route}}")], "reqps") + timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.", + [prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", + "{{http_response_status_class}}")], "reqps", stack=True) + timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.", + [prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) ' + f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")], + "percentunit", thresholds=[(0.05, "red")], minimum=0) + line.end() + line = Line(8) + timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.", + quantiles("http_server_request_duration_seconds", API), "s") + timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.", + [prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))", + "{{http_route}}")], "s") + timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.", + [prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short") + line.end() + + row("Фоновая обработка: очередь, worker, время до ready") + line = Line(8) + timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).", + [prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")], + "ops", stack=True) + timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.", + quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s") + timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).", + quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s", + thresholds=[(30, "red")]) + line.end() + line = Line(8) + timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.", + [prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"), + prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short") + timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.", + [prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"), + prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s", + thresholds=[(30, "red")]) + timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.", + [prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"), + prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")], + "ops") + line.end() + + row("Насыщение: пул БД, worker, CPU/RAM") + line = Line(8) + timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).", + [prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"), + prom("max(db_client_connection_max)", "max per replica"), + prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short") + timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).", + [prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))", + "p95 {{instance}}")], "s") + timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.", + [prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit", + thresholds=[(0.8, "orange"), (1, "red")], minimum=0) + line.end() + line = Line(8) + timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.", + [prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")], + "short") + timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.", + [prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)", + "{{job}} {{instance}}")], "percentunit") + timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).", + [prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"), + prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes") + line.end() + + row("Конвейер телеметрии: Collector и SDK") + line = Line(8) + timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.", + [prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"), + prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"), + prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"), + prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"), + prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")], + "ops") + timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.", + [prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"), + prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"), + prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"), + prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")], + "ops") + timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.", + [prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")], + "percentunit", minimum=0) + line.end() + line = Line(8) + timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.", + [prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes") + timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).", + [prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"), + prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"), + prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"), + prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")], + "ops") + line.end() + + row("Логи и трассы") + line = Line(9) + _panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)", + "description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.", + "datasource": LOKI, "gridPos": line.place(12), + "targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range", + "expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]), + "options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", + "enableLogDetails": True}}) + _panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)", + "description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.", + "datasource": TEMPO, "gridPos": line.place(12), + "targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql", + "query": "{ status = error }", "limit": 20, "tableType": "traces"}]), + "fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}}) + line.end() + + return { + "uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"], + "timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1, + "refresh": "10s", "time": {"from": "now-30m", "to": "now"}, + "templating": {"list": []}, "annotations": {"list": [{ + "builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]}, + "panels": _panels, + } + + +if __name__ == "__main__": + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels") diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..849974e --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,157 @@ +import base64 +import hashlib +import json +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore + + +def bucket_name() -> str: + """S3_BUCKET из .env (если задан), иначе значение по умолчанию.""" + env = ROOT / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("S3_BUCKET="): + return line.split("=", 1)[1].strip() + return "messenger-attachments" + + +def s3_config(bucket: str) -> str: + """Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket. + + Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects) + с суффиксом : действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет. + Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются. + """ + def read(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + config = {"identities": [ + {"name": "admin", + "credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ]} + return json.dumps(config, indent=2) + + +BROKER_VHOST = "messenger" +# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей). +EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1" +DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq" + + +def rabbit_hash(password: str) -> str: + """Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль)).""" + salt = secrets.token_bytes(4) + return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii") + + +def rabbit_definitions(passwords: dict) -> str: + """Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf). + + Права (configure / write / read, регулярные выражения по именам ресурсов): + dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет; + worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет. + Ни у кого из сервисов нет тега administrator и права создать/удалить очередь. + """ + queue_args = { + "x-queue-type": "quorum", + "x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ + "x-dead-letter-exchange": DLX, + "x-dead-letter-routing-key": DLQ_ROUTING_KEY, + "x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish) + "x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит + "x-max-length": 100000, + } + definitions = { + "users": [{"name": name, "password_hash": rabbit_hash(password), + "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} + for name, password in passwords.items()], + "vhosts": [{"name": BROKER_VHOST}], + "permissions": [ + {"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$", + "write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"}, + {"user": "worker", "vhost": BROKER_VHOST, "configure": "^$", + "write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"}, + ], + "exchanges": [ + {"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + {"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + ], + "queues": [ + {"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args}, + # DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный + # просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет. + {"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, + "arguments": {"x-queue-type": "classic"}}, + ], + "bindings": [ + {"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue", + "routing_key": ROUTING_KEY, "arguments": {}}, + {"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue", + "routing_key": DLQ_ROUTING_KEY, "arguments": {}}, + ], + } + return json.dumps(definitions, indent=2) + + +def broker_secrets(created: list, host: str = "rabbitmq") -> None: + if (SECRETS / "rabbitmq_definitions.json").exists(): + return + passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)} + for name, password in passwords.items(): + write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}") + write("rabbitmq_definitions.json", rabbit_definitions(passwords)) + created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"] + + +def main() -> None: + """Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes).""" + SECRETS.mkdir(exist_ok=True) + created = [] + if not (SECRETS / "postgres_password").exists(): + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + created += ["postgres_password", "database_url", "cache_url", "redis_auth"] + if not (SECRETS / "s3_root_password").exists(): + # admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init + write("s3_root_user", "root-" + secrets.token_hex(6)) + write("s3_root_password", secrets.token_hex(24)) + created += ["s3_root_user", "s3_root_password"] + if not (SECRETS / "s3_secret_access_key").exists(): + # отдельный ключ API с политикой только на свой bucket (создаёт s3-init) + write("s3_access_key_id", "api-" + secrets.token_hex(8)) + write("s3_secret_access_key", secrets.token_hex(24)) + created += ["s3_access_key_id", "s3_secret_access_key"] + if not (SECRETS / "s3_config.json").exists(): + write("s3_config.json", s3_config(bucket_name())) + created.append("s3_config.json") + broker_secrets(created) + if not (SECRETS / "grafana_admin_password").exists(): + # lab6: пароль администратора Grafana (GF_SECURITY_ADMIN_PASSWORD__FILE), логин admin + write("grafana_admin_password", secrets.token_urlsafe(18)) + created.append("grafana_admin_password") + if created: + print("Created in secrets/: " + ", ".join(created) + " (values are not printed).") + else: + print("secrets/ is complete, nothing changed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..3187e56 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,15 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, Prometheus, Loki, Tempo), then otel-collector, +rem api, janitor, dispatcher, worker, grafana, +rem then the balancer. Volumes are kept. +rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +cd /d "%~dp0.." +docker compose restart postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1 +docker compose restart otel-collector api janitor dispatcher worker grafana || exit /b 1 +docker compose restart lb || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps otel-collector api lb dispatcher worker grafana || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..08e5523 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ; lab6: Prometheus, Loki, Tempo), then +# otel-collector, api, janitor, dispatcher, +# worker, then the balancer. Volumes are kept (no down -v), nothing is re-seeded. +# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis s3 rabbitmq prometheus loki tempo +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq prometheus loki tempo +docker compose restart otel-collector api janitor dispatcher worker grafana +docker compose restart lb +docker compose up -d --wait --no-recreate --no-build --no-deps otel-collector api lb dispatcher worker grafana diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..8ffd0fa --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..cf4d724 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..5cdcce7 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/worker.cmd b/scripts/worker.cmd new file mode 100644 index 0000000..9ffab68 --- /dev/null +++ b/scripts/worker.cmd @@ -0,0 +1,8 @@ +@echo off +rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept). +cd /d "%~dp0.." +if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% ) +if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% ) +if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% ) +echo usage: %0 stop^|start^|status +exit /b 2 diff --git a/scripts/worker.sh b/scripts/worker.sh new file mode 100755 index 0000000..cab7c76 --- /dev/null +++ b/scripts/worker.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Demo hook for the defense: stop or start the worker without touching volumes. +# scripts/worker.sh stop | start | status +set -eu +cd "$(dirname "$0")/.." +case "${1:-}" in + stop) docker compose stop worker ;; + start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;; + status) docker compose ps worker ;; + *) echo "usage: $0 stop|start|status" >&2; exit 2 ;; +esac diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..e4cf3eb --- /dev/null +++ b/settings.py @@ -0,0 +1,111 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket. +S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333") +S3_REGION = env_str("S3_REGION", "us-east-1") +S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1) +S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5) +S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2) +S3_POOL_MAX = env_int("S3_POOL_MAX", 20) + +# Вложения: лимиты из контракта и защита памяти/времени. +MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) +DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2) +UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads") + +# Очистка незавершённых загрузок (janitor.py). +ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600) +JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + +# lab5: брокер (RabbitMQ) и фоновая обработка вложений. +# У API адреса брокера нет вообще: API пишет задание в outbox (PostgreSQL), публикует dispatcher. +BROKER_URL = read_secret("BROKER_URL_FILE", "BROKER_URL") +BROKER_HEARTBEAT_SECONDS = env_int("BROKER_HEARTBEAT_SECONDS", 30) +PIPELINE_VERSION = env_int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = env_int("WORKER_CONCURRENCY", 2) # потоков-потребителей в одном worker +WORKER_PREFETCH = env_int("WORKER_PREFETCH", 1) # неподтверждённых сообщений на поток +JOB_MAX_ATTEMPTS = env_int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = env_int("JOB_TIMEOUT_SECONDS", 60) # жёсткий предел одной обработки изображения +JOB_LEASE_SECONDS = env_int("JOB_LEASE_SECONDS", 15) # аренда processing, продлевается каждые lease/3 +JOB_BACKOFF_BASE_SECONDS = env_int("JOB_BACKOFF_BASE_SECONDS", 2) +JOB_BACKOFF_MAX_SECONDS = env_int("JOB_BACKOFF_MAX_SECONDS", 60) +JOB_RECONCILE_SECONDS = env_int("JOB_RECONCILE_SECONDS", 120) # queued без сообщения дольше -> опубликовать заново +WORKER_MEMORY_LIMIT_MB = env_int("WORKER_MEMORY_LIMIT_MB", 640) # RLIMIT_AS процесса обработки +WORKER_TMP_DIR = env_str("WORKER_TMP_DIR", "/tmp/work") +DISPATCHER_POLL_SECONDS = env_int("DISPATCHER_POLL_SECONDS", 1) +DISPATCHER_BATCH = env_int("DISPATCHER_BATCH", 100) +STAGING_RETENTION_SECONDS = env_int("STAGING_RETENTION_SECONDS", 86400) # исходник failed-вложения для replay +OUTBOX_RETENTION_SECONDS = env_int("OUTBOX_RETENTION_SECONDS", 7 * 86400) +ALIVE_FILE = env_str("ALIVE_FILE", "/tmp/alive") # healthcheck фоновых процессов + +# Учебное внедрение сбоев (faults.py). На стенде включено, в эксплуатации FAULT_INJECTION=0. +FAULT_INJECTION = env_str("FAULT_INJECTION", "0") == "1" +FAULTS_DIR = env_str("FAULTS_DIR", "/faults") + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY: + raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE") + if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS: + raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..b50ae35 --- /dev/null +++ b/storage.py @@ -0,0 +1,209 @@ +"""Приватное S3-хранилище (SeaweedFS) для байтов вложений. + +Раскладка ключей (lab5): + staging/<32 hex> — исходник, принятый API до ответа 202 (устойчивый staging); + attachments//v/original — опубликованный worker'ом оригинал (побайтно исходный файл); + attachments//v/thumbnail.jpg, crop.jpg, optimized.webp — варианты pipeline версии N; + originals/<32 hex> — исходники вложений lab4, worker обрабатывает их как staging. + +API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket +(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер, +имя файла пользователя в ключ не попадает. +""" +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings +import telemetry + +log = logging.getLogger("messenger.storage") + + +def _span(method: str, key: str | None = None): + """Клиентский спан S3: метод, bucket и ключ (ключи технические: staging/, attachments//...).""" + attributes = {"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": method, + "aws.s3.bucket": settings.S3_BUCKET} + if key is not None: + attributes["aws.s3.key"] = key + return telemetry.client_span(f"S3 {method}", attributes) + +ORIGINALS_PREFIX = "originals/" # lab4 +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "attachments/" +ALL_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, ORIGINALS_PREFIX) +READ_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей.""" + + +class ObjectMissing(Exception): + """Объекта нет в bucket (NoSuchKey / 404).""" + + +def _make_client(connect_timeout: float, read_timeout: float, attempts: int): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT, + region_name=settings.S3_REGION, + aws_access_key_id=settings.S3_ACCESS_KEY_ID or None, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # http://s3:8333// + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую + max_pool_connections=settings.S3_POOL_MAX, + # Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS) +# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor. +_quick_client = _make_client(1, 2, 1) + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "404", "NotFound") or status == 404 + + +def _unavailable(action: str, exc: Exception) -> StorageUnavailable: + detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else "" + log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail, + extra={"event": "s3.unavailable", "rpc.method": action, "error.type": type(exc).__name__}) + return StorageUnavailable() + + +def new_key(object_id: str) -> str: + """Ключ staging для нового исходника (имя файла пользователя в ключ не попадает).""" + return STAGING_PREFIX + object_id + + +def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + """Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект.""" + body.seek(0) + with _span("PutObject", key) as span: + span.set_attribute("http.request.body.size", size) + try: + _client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("put", exc) from None + + +def _get_body(key: str): + try: + return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"] + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing(key) from None + raise _unavailable("get", exc) from None + except BotoCoreError as exc: + raise _unavailable("get", exc) from None + + +def open_read(key: str): + """GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после. + + Спан покрывает запрос до получения заголовков; тело API стримит клиенту уже после. + """ + with _span("GetObject", key): + return _get_body(key) + + +def download(key: str, path, max_bytes: int) -> int: + """GET объекта в файл кусками. Дальше max_bytes не читаем: размер не совпадёт, worker отклонит исходник.""" + with _span("GetObject", key) as span: + stream = _get_body(key) + size = 0 + try: + with open(path, "wb") as file: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + break + size += len(chunk) + file.write(chunk) + if size > max_bytes: + break + except (BotoCoreError, OSError) as exc: + raise _unavailable("get", exc) from None + finally: + stream.close() + span.set_attribute("http.response.body.size", size) + return size + + +def iter_chunks(stream) -> Iterator[bytes]: + """Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание.""" + try: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return + yield chunk + except (BotoCoreError, OSError) as exc: + # Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length). + log.warning("S3 stream interrupted: %s", type(exc).__name__) + raise + finally: + stream.close() + + +def delete(key: str, quick: bool = False) -> None: + """Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа.""" + with _span("DeleteObject", key): + try: + (_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("delete", exc) from None + + +def exists(key: str) -> bool: + with _span("HeadObject", key): + try: + _client.head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise _unavailable("head", exc) from None + except BotoCoreError as exc: + raise _unavailable("head", exc) from None + + +def list_objects(prefix: str) -> Iterator[tuple[str, datetime, int]]: + """Все объекты под prefix: (key, last_modified, size). Постранично, по 1000.""" + paginator = _client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"], item["Size"] + except (BotoCoreError, ClientError) as exc: + raise _unavailable("list", exc) from None + + +def ping() -> None: + try: + _client.head_bucket(Bucket=settings.S3_BUCKET) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("head_bucket", exc) from None diff --git a/student-tests/check_configs.py b/student-tests/check_configs.py new file mode 100644 index 0000000..96441d0 --- /dev/null +++ b/student-tests/check_configs.py @@ -0,0 +1,107 @@ +"""Lab6: конфигурации наблюдаемости из Git проверены «родными» инструментами тех же версий + provisioning Grafana. + + python student-tests/check_configs.py + +В Docker-стенде (по умолчанию) инструменты запускаются в образах из compose.yaml: + otelcol-contrib validate, promtool check config/rules, promtool test rules (unit-тесты алертов), + loki -verify-config, tempo -config.verify; затем через HTTP API Grafana: источники данных созданы + provisioning-ом и отвечают (health), dashboard импортирован, его панели ссылаются на эти источники. +OBS_BIN_DIR=<каталог с бинарниками> — проверить конфиги локальными бинарниками без Docker (Grafana пропускается, +если GRAFANA_URL недоступен). +""" +import base64 +import json +import os +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + +from common import ROOT, check + +BIN = os.environ.get("OBS_BIN_DIR") +GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000").rstrip("/") + + +def run(title, docker_args, local_args): + if BIN: + command = [str(Path(BIN) / local_args[0]), *local_args[1:]] + else: + command = ["docker", "compose", *docker_args] + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + output = (result.stdout + result.stderr).strip() + tail = "\n".join(output.splitlines()[-6:]) + print(f"--- {title}: exit {result.returncode}\n{tail}") + check(result.returncode == 0, f"{title} failed") + + +def local_copy(path, replacements): + text = (ROOT / path).read_text(encoding="utf-8") + for old, new in replacements: + text = text.replace(old, new) + handle = tempfile.NamedTemporaryFile("w", suffix=Path(path).suffix, delete=False, encoding="utf-8") + handle.write(text) + handle.close() + return handle.name + + +def grafana(path): + password = (ROOT / "secrets/grafana_admin_password").read_text(encoding="utf-8").strip() + request = urllib.request.Request(GRAFANA_URL + path, headers={ + "Authorization": "Basic " + base64.b64encode(f"admin:{password}".encode()).decode(), "Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=10) as reply: + return json.loads(reply.read().decode("utf-8")) + + +def main(): + tmp = tempfile.mkdtemp() + run("otelcol-contrib validate (infra/otel/collector.yaml)", + ["run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"], + ["otelcol-contrib", "validate", f"--config={ROOT / 'infra/otel/collector.yaml'}"]) + run("promtool check config (infra/prometheus/prometheus.yml)", + ["exec", "-T", "prometheus", "promtool", "check", "config", "/etc/prometheus/prometheus.yml"], + ["promtool", "check", "config", local_copy("infra/prometheus/prometheus.yml", [ + ("/etc/prometheus/rules/", str(ROOT / "infra/prometheus/rules") + "/")])]) + run("promtool check rules (recording + alerts)", + ["exec", "-T", "prometheus", "sh", "-c", "promtool check rules /etc/prometheus/rules/*.yml"], + ["promtool", "check", "rules", *map(str, sorted((ROOT / "infra/prometheus/rules").glob("*.yml")))]) + run("promtool test rules (unit tests of alert thresholds/windows/for)", + ["exec", "-T", "prometheus", "promtool", "test", "rules", "/etc/prometheus/tests/alerts_test.yml"], + ["promtool", "test", "rules", str(ROOT / "infra/prometheus/tests/alerts_test.yml")]) + run("loki -verify-config (infra/loki/loki.yaml)", + ["run", "--rm", "--no-deps", "loki", "-config.file=/etc/loki/loki.yaml", "-verify-config"], + ["loki", f"-config.file={local_copy('infra/loki/loki.yaml', [('/loki', tmp + '/loki')])}", "-verify-config"]) + run("tempo -config.verify (infra/tempo/tempo.yaml)", + ["run", "--rm", "--no-deps", "tempo", "-config.file=/etc/tempo/tempo.yaml", "-config.verify=true"], + [os.environ.get("TEMPO_BIN", "tempo"), + f"-config.file={local_copy('infra/tempo/tempo.yaml', [('/var/tempo', tmp + '/tempo')])}", "-config.verify=true"]) + + try: + health = grafana("/api/health") + except (OSError, urllib.error.URLError) as exc: + check(BIN, f"Grafana is not reachable at {GRAFANA_URL}: {exc}") + print(f"--- Grafana: not reachable at {GRAFANA_URL}, skipped (local binaries mode)") + print("PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid") + return + print(f"--- Grafana {health.get('version')}: database {health.get('database')}") + sources = {d["uid"]: d for d in grafana("/api/datasources")} + for uid, kind in (("prometheus", "prometheus"), ("loki", "loki"), ("tempo", "tempo")): + check(uid in sources and sources[uid]["type"] == kind, f"datasource {uid} is not provisioned") + state = grafana(f"/api/datasources/uid/{uid}/health") + print(f"datasource {uid} ({kind}): {state.get('status')} {state.get('message', '')[:80]}") + check(state.get("status") == "OK", f"datasource {uid} is not healthy: {state}") + dashboard = grafana("/api/dashboards/uid/messenger-lab6") + panels = [p for p in dashboard["dashboard"]["panels"] if p["type"] != "row"] + used = {t.get("datasource", {}).get("uid") for p in panels for t in p.get("targets", [])} + print(f"dashboard '{dashboard['dashboard']['title']}' in folder '{dashboard['meta'].get('folderTitle')}': " + f"{len(panels)} panels, provisioned={dashboard['meta'].get('provisioned')}, datasources {sorted(used)}") + check(dashboard["meta"].get("provisioned"), "dashboard must come from provisioning, not the UI") + check(used <= set(sources), "panels reference a datasource that is not provisioned") + print("PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana " + "provisioned 3 healthy datasources and the dashboard without manual steps") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..eaea628 --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,119 @@ +"""Lab6: каждый запрос dashboard и правил выполняется против живых хранилищ. + + python student-tests/check_dashboard.py + +Проверяет, что dashboard ссылается на реально существующие метрики/метки (контракт наблюдаемости): + - каждое PromQL-выражение панелей и recording/alert rules выполняется без ошибки; + - для ключевых сигналов (RED API, задания, очередь, пул, worker, CPU/RAM) результат НЕ пуст; + - LogQL логов и TraceQL трасс выполняются; + - в метках метрик приложения нет запрещённых (user_id, chat_id, message_id, attachment_id, request_id, + url, filename, text) и нет значений-UUID (кроме instance). +Перед запуском нужен трафик: make test (smoke) — он создаёт запросы, задание и логи. +""" +import json +import re +import sys +from pathlib import Path + +import obs +from common import check + +ROOT = Path(__file__).resolve().parents[1] +DASHBOARD = ROOT / "infra/grafana/dashboards/messenger.json" +RULES = ROOT / "infra/prometheus/rules" +FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "url_full", + "filename", "text", "job_id", "trace_id", "span_id", "authorization"} +UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +# Метрики, которые обязаны иметь данные после smoke (имя -> зачем) +MUST_HAVE_DATA = { + 'http_server_request_duration_seconds_count{job="messenger-api"}': "HTTP requests/duration (3)", + "messenger_job_duration_seconds_count": "jobs processed/failed + duration (3)", + "messenger_attachment_time_to_ready_seconds_count": "end-to-end time to ready (4)", + "messenger_queue_depth": "queue depth (4)", + "messenger_queue_oldest_age_seconds": "oldest job age (4)", + "db_client_connection_count": "DB pool used/idle (4)", + "db_client_connection_pending_requests": "DB pool waiting (4)", + "messenger_worker_busy": "worker busy (4)", + "messenger_worker_capacity": "worker capacity (4)", + "process_cpu_time_seconds_total": "process CPU (4)", + "process_memory_usage_bytes": "process RAM (4)", + "otelcol_exporter_queue_size": "collector queue (4)", +} + + +def substitute(expr): + return expr.replace("$__rate_interval", "1m").replace("$__interval", "1m").replace("$__range", "1h") + + +def main(): + dashboard = json.loads(DASHBOARD.read_text(encoding="utf-8")) + prom_exprs, loki_exprs, tempo_exprs = [], [], [] + for panel in dashboard["panels"]: + for target in panel.get("targets", []): + uid = target.get("datasource", {}).get("uid") + if uid == "prometheus": + prom_exprs.append((panel["title"], target["expr"])) + elif uid == "loki": + loki_exprs.append((panel["title"], target["expr"])) + elif uid == "tempo": + tempo_exprs.append((panel["title"], target["query"])) + try: + import yaml # noqa: F401 — если PyYAML есть, проверим и выражения правил + rules = [] + for path in sorted(RULES.glob("*.yml")): + for group in yaml.safe_load(path.read_text(encoding="utf-8"))["groups"]: + for rule in group["rules"]: + rules.append((f"{path.name}: {rule.get('record') or rule.get('alert')}", rule["expr"])) + except ImportError: + rules = [] + + empty, errors = [], [] + for title, expr in prom_exprs + rules: + try: + rows = obs.prom_query(substitute(expr)) + except Exception as exc: # noqa: BLE001 + errors.append(f"{title}: {exc}") + continue + if not rows: + empty.append(title) + print(f"PromQL: {len(prom_exprs)} dashboard expressions + {len(rules)} rule expressions executed, " + f"{len(errors)} errors, {len(empty)} empty now (no traffic of that kind / Docker-only / no incident)") + for title in sorted(set(empty)): + print(f" empty now: {title}") + check(not errors, "PromQL errors:\n" + "\n".join(errors)) + + for title, expr in loki_exprs: + obs.loki_query(expr, minutes=60, limit=5) + print(f"LogQL ok: {title}: {expr}") + for title, query in tempo_exprs: + found = obs.tempo_search(query, minutes=60, limit=5) + print(f"TraceQL ok: {title}: {query} -> {len(found)} trace(s)") + + print() + for expr, why in MUST_HAVE_DATA.items(): + rows = obs.prom_query(expr) + check(rows, f"no data for {expr} ({why})") + print(f"data: {expr:<70} {len(rows):>3} series <- {why}") + + print() + bad = [] + for job in ("messenger-api", "messenger-worker", "messenger-dispatcher", "messenger-janitor"): + for series in obs.prom_series(f'{{job="{job}"}}'): + for name, value in series.items(): + if name in FORBIDDEN_LABELS: + bad.append(f"{series['__name__']}: forbidden label {name}") + if name != "instance" and UUID.search(value): + bad.append(f"{series['__name__']}: UUID value in label {name}") + labels = sorted({name for job in ("messenger-api", "messenger-worker", "messenger-dispatcher") + for series in obs.prom_series(f'{{job="{job}"}}') for name in series}) + print(f"labels used by application metrics: {', '.join(labels)}") + check(not bad, "high-cardinality labels:\n" + "\n".join(sorted(set(bad))[:20])) + route_values = sorted({row[0].get("http_route") for row in obs.prom_query( + 'count by (http_route) (http_server_request_duration_seconds_count{job="messenger-api"})')}) + print(f"http_route values (templates, not URLs): {route_values}") + check(all(not UUID.search(value or "") for value in route_values), "raw URL in http_route") + print("PASS: every dashboard/rule query runs, key signals have data, no forbidden labels") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 0000000..2cf933b --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,118 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + for one_shot in ("migrate", "s3-init"): + status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip() + check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis", "s3", "rabbitmq"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + if service in ("janitor", "worker", "dispatcher"): + # lab6: + внутренняя сеть telemetry (только до Collector), портов нет + check(not ports and networks == ["data", "telemetry"], f"{service}: networks data+telemetry, no ports") + if service == "otel-collector": + check(not ports and networks == ["telemetry"], "otel-collector: only the internal telemetry network, no ports") + if service in ("prometheus", "loki", "tempo", "grafana"): + check(ports and all(p.startswith("127.0.0.1:") for p in ports), f"{service}: ports only on 127.0.0.1") + check("data" not in networks and "edge" not in networks, f"{service} must not reach app data/edge networks") + check(api_count >= 2, "need at least 2 api replicas") + running = {r["Config"]["Labels"].get("com.docker.compose.service") for r in rows if r["State"]["Running"]} + check({"rabbitmq", "worker", "dispatcher"} <= running, "lab5: broker, worker and dispatcher must be running") + check(any(v.get("target") == "/var/lib/rabbitmq" and v.get("type") == "volume" + for v in services["rabbitmq"].get("volumes", [])), "rabbitmq needs a persistent volume") + check(services["rabbitmq"].get("hostname") == "rabbitmq", "rabbitmq needs a fixed hostname (node data dir)") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + # lab6: наблюдаемость + check(config["networks"]["telemetry"].get("internal") is True, "telemetry network must be internal") + check({"otel-collector", "prometheus", "loki", "tempo", "grafana"} <= running, + "lab6: collector, prometheus, loki, tempo and grafana must be running") + for name, target in (("prometheus", "/prometheus"), ("loki", "/loki"), ("tempo", "/var/tempo"), + ("grafana", "/var/lib/grafana")): + check(any(v.get("target") == target and v.get("type") == "volume" for v in services[name].get("volumes", [])), + f"{name} needs a persistent volume at {target}") + for name, service in services.items(): + image = service.get("image", "") + check(not image.endswith(":latest") and (":" in image or not image), f"{name}: image version must be pinned") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(not services["otel-collector"].get("secrets"), "otel-collector gets no secrets") + check({s["source"] for s in services["grafana"].get("secrets", [])} == {"grafana_admin_password"}, + "grafana: only its admin password") + for name in ("api", "worker", "dispatcher", "janitor", "prometheus", "loki", "tempo"): + check("grafana_admin_password" not in {s["source"] for s in services[name].get("secrets", [])}, + f"{name} must not get the Grafana password") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + for name in ("api", "janitor", "worker"): + names = {s["source"] for s in services[name]["secrets"]} + check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys") + check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key") + broker_secrets = {"broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions"} + for name in ("api", "janitor", "lb", "migrate"): + names = {s["source"] for s in services[name].get("secrets", [])} + check(not names & broker_secrets, f"{name} must not get broker credentials (API uses the outbox)") + check({s["source"] for s in services["dispatcher"]["secrets"]} == {"database_url", "broker_url_dispatcher"}, + "dispatcher: only database_url and its own broker user") + check(not {s["source"] for s in services["dispatcher"]["secrets"]} & {"s3_access_key_id", "s3_secret_access_key"}, + "dispatcher needs no S3 key") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, postgres/redis/s3/rabbitmq only in the internal data network, worker/dispatcher/janitor in " + f"data+telemetry, collector only in the internal telemetry network, prometheus/loki/tempo/grafana on " + f"127.0.0.1 with persistent volumes, pinned images, S3 admin keys only in s3 and s3-init, broker " + f"credentials only in dispatcher/worker") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 0000000..f57e649 --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,449 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") # по умолчанию evidence/ +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test) with telemetry ON, with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + # ---- lab6 ---- + ("configs", "lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning", + [PY, "student-tests/check_configs.py"]), + ("telemetry", "lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets", + [PY, "student-tests/test_telemetry.py", "request", "upload-chain", "retry-chain", "replay-link"]), + ("dashboard", "lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels", + [PY, "student-tests/check_dashboard.py"]), + ("telemetry-history", "lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes)", + [PY, "student-tests/test_telemetry.py", "history"]), + ("incident-worker", "lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved", + [PY, "student-tests/incidents.py", "worker-stop"]), + ("incident-db", "lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved", + [PY, "student-tests/incidents.py", "db-slow"]), + ("incident-telemetry", "lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted", + [PY, "student-tests/incidents.py", "telemetry-down"]), + ("incident-collector", "lab6: Collector down -> API unaffected, SDK queue bounded, drops counted", + [PY, "student-tests/incidents.py", "collector-down"]), + ("jobs-variants", "lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped", + [PY, "student-tests/test_jobs.py", "variants"]), + ("jobs-limits", "lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)", + [PY, "student-tests/test_jobs.py", "limits"]), + ("jobs-recovery", "lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits", + [PY, "student-tests/test_jobs.py", "worker-down", "broker-down"]), + ("jobs-outbox", "lab5: dispatcher crash between COMMIT and publish / after confirm", + [PY, "student-tests/test_jobs.py", "outbox-crash"]), + ("jobs-crash", "lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease", + [PY, "student-tests/test_jobs.py", "crash-after-put", "crash-after-commit", "staging", "lease"]), + ("jobs-retry", "lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ", + [PY, "student-tests/test_jobs.py", "retry", "exhausted-replay", "permanent", "dlq"]), + ("jobs-janitor-replay", "lab5: janitor does not delete objects re-written by replay (review fix)", + [PY, "student-tests/test_janitor_replay.py"]), + ("jobs-concurrency", "lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing", + [PY, "student-tests/test_jobs.py", "concurrency"]), + ("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids", + [PY, "student-tests/test_attachments.py"]), + ("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap", + [PY, "student-tests/test_s3_security.py"]), + ("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage", + [PY, "student-tests/test_consistency.py"]), + ("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas", + [PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]), + ("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)", + [PY, "student-tests/run_failover.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, " + f"janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: " + f"otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB.") + lines.append("") + lines += lab6_measurements() + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + memory = load_json("upload-memory.json") + if memory: + lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики " + f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). " + f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:") + lines.append("") + lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | " + "Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for run in memory["runs"]: + for n, r in enumerate(run["replicas"]): + head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | " + f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |" + cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-" + lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | " + f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |") + naive = memory["single_file_extra_rss_mib"] + lines.append("") + lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в " + f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка " + f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в " + f"{memory['api_limit_mib']} MiB.") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и " + f"прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + consistency = EVIDENCE / "consistency.txt" + if consistency.exists(): + text = consistency.read_text(encoding="utf-8") + timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text) + if len(timings) >= 2: + impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, " + f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | " + f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |") + impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за " + f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |") + if "PASS: S3 written but DB not committed" in text: + impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | " + "вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | " + "GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | " + "мусора не осталось; повторный проход ничего не делает |") + if "PASS: an interrupted upload leaves no row" in text: + impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | " + "строк и объектов не прибавилось, временных файлов 0 | мгновенно |") + impacts += lab5_impacts() + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def lab6_measurements(): + """Таблицы lab6 из evidence/incident-*.json (их пишет student-tests/incidents.py).""" + lines = [] + worker, db, tel, col = (load_json(f"incident-{n}.json") for n in ("worker-stop", "db-slow", "telemetry-down", + "collector-down")) + if worker or db or tel: + lines.append("Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):") + lines.append("") + lines.append("| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |") + lines.append("| --- | --- | --- | --- | --- |") + if worker: + for name, seconds in worker["firing_after_stop_s"].items(): + lines.append(f"| 1. worker | `docker compose stop worker` + 3 загрузки | {name} | {seconds} (от stop) | " + f"{worker['resolved_after_start_s'].get(name)} (от start) |") + if db: + lines.append(f"| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на {db['lock_seconds']} с под " + f"нагрузкой 8 потоков | MessengerHighErrorRate | {db['firing_after_lock_s']} (от LOCK) | " + f"{db['resolved_after_unlock_s']} (от COMMIT блокировки) |") + if tel: + lines.append(f"| 3. телеметрия | `docker compose stop {' '.join(tel['stopped'])}` на {tel['outage_s']} с | " + f"TelemetryExportFailing | {tel.get('TelemetryExportFailing_firing_at_s')} (от stop) | " + f"{tel['resolved_after_start_s']} (от start) |") + lines.append("") + if worker: + d = worker["diagnosis"] + lines.append(f"Инцидент 1, наблюдения в момент firing: consumers очереди = {d['queue_consumers']}, сообщений в " + f"RabbitMQ = {d['broker_queue_depth']}, queued заданий = {d['jobs_backlog_queued']}, возраст " + f"старейшего = {d['oldest_job_age_s']} с, возраст outbox = {d['outbox_oldest_age_s']} с (публикация " + f"работает), метрики worker не обновлялись {d['worker_metrics_last_seen_s']} с, последний лог worker: " + f"event=`{(d.get('last_worker_log') or {}).get('event')}`; трасса загрузки содержит `publish attachments`, " + f"но не содержит `process attachments.process.v1`. После start: вложения ready через " + f"{max(worker['ready_after_start_s'])} с, backlog {worker.get('after_recovery', {}).get('jobs_backlog_queued')}.") + lines.append("") + if db: + d, e, c = db["diagnosis"], db["diagnosis"]["error_trace"], db["diagnosis"]["handler_cpu_contrast"] + lines.append(f"Инцидент 2, наблюдения: доля 5xx (1m) = {d['http_5xx_ratio_1m']}, коды POST /messages за 1m = " + f"{d['status_codes_1m']}, p95 POST /messages = {d['http_p95_messages_s']} с, чтения других таблиц " + f"(GET /users/me, /chats) во время блокировки {d.get('reads_during_lock_ms')} мс, занято соединений " + f"пула = {d['pool_used']}, p95 ожидания пула = " + f"{d['pool_wait_p95_s']} с, логи API event=db.unavailable по error_type: " + f"{d['api_log_db_unavailable_by_error_type']}. Error-трасса {e['trace_id']}: server-спан " + f"{e['server_ms']} мс, из них `{e['slowest_db_span']}` {e['slowest_db_ms']} мс (доля " + f"{e['db_share']}, статус {e['db_span_status']}). Контраст «медленный обработчик» (трасса " + f"{c['trace_id']}, POST /users): server {c['server_ms']} мс, `argon2id hash` {c['argon2id_ms']} мс, " + f"все спаны БД {c['db_ms_total']} мс. Нагрузка: до {db['baseline']['success_rps']} успешных записей/с, " + f"во время блокировки {db['during_lock']['codes']}, после {db['after']['success_rps']}/с без ошибок.") + lines.append("") + if tel or col: + lines.append("| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |") + lines.append("| --- | --- | --- | --- |") + if tel: + probes = tel["probe_traces"] + lost = [p["sent_at_s"] for p in probes if not p["trace_in_tempo"]] + lines.append(f"| `stop {' '.join(tel['stopped'])}` на {tel['outage_s']} с (окно повторов Collector 120 с) | " + f"{tel['during']['requests']} запросов, коды {tel['during']['codes']}, доля успеха " + f"{tel['during']['success_ratio']} | heap Collector {tel['collector_heap_before_mib']} -> max " + f"{tel['collector_heap_max_mib']} MiB (limit 200), контейнер max {tel['collector_container_max_mib']} MiB | " + f"send/enqueue failed в Collector: {tel['lost_in_collector']}; пробные трассы дошли " + f"{len(probes) - len(lost)}/{len(probes)}, потеряны отправленные на {lost} с |") + if col: + lines.append(f"| `stop otel-collector` на {col['outage_s']} с | {col['during']['requests']} запросов, коды " + f"{col['during']['codes']}, доля успеха {col['during']['success_ratio']} | api {col['api_mib_before']} " + f"-> max {col['api_mib_max']} MiB (2 реплики) | SDK api потерял спанов: {col['sdk_lost_spans_api']} " + f"(queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start " + f"экспорт возобновился ({col['sdk_exported_ok_after_restart']} спанов за минуту) |") + lines.append("") + return lines + + +def lab5_line(name, pattern): + """Строка вывода test_jobs.py из evidence/.txt, содержащая pattern (или None).""" + path = EVIDENCE / f"{name}.txt" + if not path.exists(): + return None + for line in path.read_text(encoding="utf-8").splitlines(): + if pattern in line and not line.startswith(("#", "---")): + return line.strip() + return None + + +def lab5_impacts(): + """Строки таблицы отказов lab5: наблюдение — дословно строка вывода теста (числа из прогона).""" + rows = [ + ("jobs-recovery", "worker started", "worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker", + "202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready", + "потерь нет; staging удалён после COMMIT"), + ("jobs-recovery", "rabbitmq started", "rabbitmq остановлен во время upload", + "202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется", "потерь нет"), + ("jobs-outbox", "killed between COMMIT and publish", "crash dispatcher между COMMIT задания и publish", + "строка outbox не отмечена -> публикуется после рестарта процесса", "без ручного действия"), + ("jobs-outbox", "after publisher confirm", "crash dispatcher после confirm, до отметки outbox", + "дубль сообщения, worker распознаёт по состоянию job", "один набор вариантов"), + ("jobs-crash", "killed after S3 PUT", "crash worker после PUT в S3, до COMMIT и ack", + "повтор после истечения аренды перезаписывает те же ключи", "один набор, 4 объекта"), + ("jobs-crash", "after COMMIT, before ack", "crash worker после COMMIT, до ack", + "redelivery ready-задачи -> no-op ack", "очередь пуста, дублей нет"), + ("jobs-crash", "frozen attempt", "worker завис (не продлевает аренду)", + "dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing", "ready не откатан"), + ("jobs-limits", "imaging hang", "зависание обработки изображения", + "процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась", "повтор -> ready"), + ("jobs-limits", "memory bomb", "нехватка памяти в процессе обработки (RLIMIT_AS)", + "failed/resource_limit, worker жив", "постоянная ошибка, без повторов"), + ("jobs-retry", "storage_unavailable ->", "2 временные ошибки S3", "backoff с jitter, затем ready", "-"), + ("jobs-retry", "retry_exhausted (public", "5 временных ошибок подряд", + "failed/retry_exhausted, реестр failed, content 409", "replay оператора"), + ("jobs-retry", "ops replay", "replay failed и повторный replay ready", "тот же job_id -> ready; повтор отклонён", + "дубля нет"), + ("jobs-retry", "-> reject -> DLQ", "невалидное сообщение и неизвестный job_id", "reject -> DLQ, dlq-replay не исполняет", + "сообщения сохранены"), + ("jobs-concurrency", "peak processing", "8 заданий в очереди, старт worker", + "WORKER_CONCURRENCY=2, prefetch 1", "-"), + ] + out = [] + for name, pattern, what, expected, recovery in rows: + line = lab5_line(name, pattern) + if line: + out.append(f"| {what} | {expected} | {line.replace('|', '/')} | {recovery} |") + return out + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed and os.environ.get("REPORT_UPDATE", "1") == "1": + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..66cc2a8 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,347 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None): + """POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся.""" + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body=body, headers={"Idempotency-Key": key or new_key()}) + + +def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=202, timeout=60, + wait=True): + """POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра. + + Lab5: успешный приём — 202 со снимком queued (переход 201 -> 202 из COURSE.md). При wait=True + ответ дополняется ожиданием готовности: .json — метаданные после GET в состоянии ready/failed, + .accepted — исходный снимок 202. Так проверки lab4 («файл скачивается побайтно») сохраняются. + """ + query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="") + headers = {"Content-Type": content_type} if content_type is not None else {} + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data, + token=user["token"] if user else None, headers=headers, timeout=timeout) + if reply.status == 202: + check(reply.json["status"] == "queued" and reply.json["variants"] == [], "202 must be a queued snapshot") + reply.accepted = reply.json + if wait: + reply.json = wait_attachment(user, reply.json["id"]) + return reply + + +def wait_attachment(user, attachment_id, statuses=("ready", "failed"), seconds=60): + """GET метаданных, пока вложение не перейдёт в одно из statuses (smoke ждёт ready до 60 с).""" + deadline = time.monotonic() + seconds + while True: + att = call("GET", f"/api/v1/attachments/{attachment_id}", 200, token=user["token"]).json + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"attachment {attachment_id} is {att['status']} after {seconds} s") + time.sleep(0.2) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def app_python(code, *args, service="api", env=None): + """Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API). + + Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON). + """ + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return output + + +def janitor(*args, env=None): + """Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт.""" + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += ["janitor", "python", "janitor.py", "--once", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return json.loads(output.splitlines()[-1]) + + +def sql_value(query): + """Одно значение из PostgreSQL (psql -A -t).""" + code, output, error = psql(query) + check(code == 0, f"psql failed: {error[-300:]}") + return output + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 0000000..8f97f1c --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..81416de --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 0000000..01cd9e7 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,76 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY m.created_at, m.id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 0000000..19421a1 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 0000000..38faa2b Binary files /dev/null and b/student-tests/fixtures/progressive.jpg differ diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000..6816ff5 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..e49bfc6 --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,93 @@ +"""Синтетические изображения для тестов вложений: только стандартная библиотека. + +PNG собирается вручную (сигнатура, IHDR, IDAT, IEND), поэтому можно получить любой размер +в пикселях и байтах, а также испортить конкретный байт. JPEG берётся из fixtures/. +""" +import os +import struct +import zlib +from pathlib import Path + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +ROOT = Path(__file__).resolve().parents[1] +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: + """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). + + pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах. + idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). + """ + channels = 1 if gray else 3 + header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace) + if idat is None: + compressor = zlib.compressobj(6) + parts = [] + # Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход. + passes = [(width, height)] if not interlace else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + for pass_width, pass_height in passes: + if pass_width <= 0 or pass_height <= 0: + continue + row = b"\x00" + bytes(pass_width * channels) + for _ in range(pass_height): + parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row)) + parts.append(compressor.flush()) + idat = b"".join(parts) + body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) + if pad_to is not None: + need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0" + assert need >= 0, "image is already larger than pad_to" + body += chunk(b"tEXt", b"Comment\x00" + b"a" * need) + return body + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями).""" + side = int(((target_bytes - 4096) / 3) ** 0.5) + return png(side, side, noise=True) + + +def sample_png() -> bytes: + return (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (FIXTURES / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (FIXTURES / "progressive.jpg").read_bytes() + + +def flip_byte(data: bytes, position: int) -> bytes: + return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:] + + +GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + + +# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) --- + +def ihdr(width, height, depth=8, color=2, interlace=0): + return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + + +def zero_idat(width, height, depth=8, channels=3): + """Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный).""" + row_bytes = (width * depth * channels + 7) // 8 + return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height) + + +def assemble(*chunks): + """PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан.""" + if not chunks or chunks[-1][0] != b"IEND": + chunks = chunks + ((b"IEND", b""),) + return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks) diff --git a/student-tests/incidents.py b/student-tests/incidents.py new file mode 100644 index 0000000..b89f607 --- /dev/null +++ b/student-tests/incidents.py @@ -0,0 +1,499 @@ +"""Lab6, уровень 4: контролируемые инциденты, алерты firing -> resolved и разбор причины по телеметрии. + + python student-tests/incidents.py worker-stop # (1) остановлен worker: растёт возраст задач + python student-tests/incidents.py db-slow # (2) PostgreSQL медленный (блокировка таблицы) + python student-tests/incidents.py telemetry-down # (3) недоступны Tempo и Loki + python student-tests/incidents.py collector-down # (3') недоступен сам Collector + python student-tests/incidents.py # все по очереди (~20 минут) + +Каждый сценарий: нагрузка/загрузки -> отказ -> ожидание firing в Prometheus -> «диагностика» только по +сохранённым наблюдениям (метрики, логи, трассы) -> восстановление -> ожидание resolved. Хронология +печатается, итог пишется в evidence/incident-<имя>.json (без токенов и текстов). Тест сам возвращает +стенд в рабочее состояние (start/unlock), даже если упал по проверке. +""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid + +import obs +from common import ROOT, call, check, compose, new_chat, new_user, try_call, wait_attachment +from images import png + +EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") +FIRE_TIMEOUT = 300 +RESOLVE_TIMEOUT = 300 +T0 = time.monotonic() + + +def say(text): + print(f"[{time.monotonic() - T0:6.1f}s] {text}", flush=True) + + +def save(name, data): + EVIDENCE.mkdir(exist_ok=True) + path = EVIDENCE / f"incident-{name}.json" + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + say(f"saved {path}") + + +def q(expr, default=None): + try: + value = obs.prom_value(expr, default) + except Exception: # noqa: BLE001 — Prometheus может быть недоступен в сценарии + return default + return default if value is None or value != value else value # NaN (нет данных) -> default + + +def rnd(value, digits=3): + return None if value is None else round(value, digits) + + +class Load: + """Closed-loop POST сообщений в N потоков. Хранит (время, код или None, задержка) — без тела ответов.""" + + def __init__(self, threads=4, think=0.05, timeout=12): + self.user = new_user("incident") + self.chat = new_chat(self.user, title="Incident load") + self.threads, self.think, self.timeout = threads, think, timeout + self.samples = [] + self.lock = threading.Lock() + self.stop_event = threading.Event() + self.workers = [] + + def _run(self): + while not self.stop_event.is_set(): + started = time.monotonic() + reply = try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", body={"text": "load"}, + token=self.user["token"], headers={"Idempotency-Key": uuid.uuid4().hex}, + timeout=self.timeout) + with self.lock: + self.samples.append((started, reply.status if reply else None, time.monotonic() - started)) + self.stop_event.wait(self.think) + + def start(self): + for _ in range(self.threads): + thread = threading.Thread(target=self._run, daemon=True) + thread.start() + self.workers.append(thread) + return self + + def stop(self): + self.stop_event.set() + for thread in self.workers: + thread.join(timeout=self.timeout + 2) + + def window(self, since, until=None): + until = until or time.monotonic() + with self.lock: + rows = [s for s in self.samples if since <= s[0] < until] + codes = {} + for _, status, _ in rows: + key = str(status) if status else "timeout/conn" + codes[key] = codes.get(key, 0) + 1 + ok = [lat for _, status, lat in rows if status and status < 500] + ok.sort() + p95 = ok[int(len(ok) * 0.95) - 1] if ok else None + seconds = max(until - since, 0.001) + return {"requests": len(rows), "codes": codes, "success_ratio": rnd(len(ok) / len(rows), 4) if rows else None, + "success_rps": rnd(len(ok) / seconds, 1), "p95_ms_successful": rnd(p95 * 1000 if p95 else None, 1)} + + +def container_mem_mib(service): + """Память контейнера (docker stats) — работает и когда Prometheus/Collector недоступны.""" + ids = subprocess.run(["docker", "compose", "ps", "-q", service], cwd=ROOT, capture_output=True, + text=True).stdout.split() + total = 0.0 + for cid in ids: + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}", cid], cwd=ROOT, + capture_output=True, text=True).stdout.strip() + if not out: + continue + data = json.loads(out.splitlines()[0]) + if "MemUsageBytes" in data: + total += data["MemUsageBytes"] / 2**20 + continue + used = data.get("MemUsage", "0B").split("/")[0].strip() + for unit, factor in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("kB", 1 / 1024), ("MB", 1), ("GB", 1024), + ("B", 1 / 2**20)): + if used.endswith(unit): + total += float(used[:-len(unit)]) * factor + break + return round(total, 1) + + +def traced_upload(user, chat_id, size=(64, 48)): + trace_id, traceparent = obs.new_traceparent() + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename=incident.png", 202, raw=png(*size), + token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent}) + return reply.json["id"], trace_id + + +def probe(user, chat_id): + """Запрос с известным trace_id: после восстановления проверим, дошла ли его трасса до Tempo.""" + trace_id, traceparent = obs.new_traceparent() + reply = try_call("POST", f"/api/v1/chats/{chat_id}/messages", body={"text": "probe"}, token=user["token"], + headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}, timeout=10) + return trace_id, reply.status if reply else None + + +def counter(expr): + return q(f"sum({expr})", 0.0) or 0.0 + + +# --------------------------------------------------------------------------------------------------- +def worker_stop(): + """(1) Остановлен worker. Ожидаем: MessengerWorkerNoConsumers и MessengerQueueLagGrowing -> firing; + причина находится по метрикам очереди, логам и обрыву трасс; после start backlog -> 0, resolved.""" + result = {"scenario": "worker-stop", "date": time.strftime("%Y-%m-%d %H:%M:%S")} + user = new_user("lag") + chat = new_chat(user, title="Lag chat") + check(obs.alerts().get("MessengerQueueLagGrowing") is None, "MessengerQueueLagGrowing is already active") + say("baseline: consumers=%s, oldest job age=%s s" % ( + q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'), + q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'))) + stopped = time.monotonic() + compose("stop", "worker") + say("docker compose stop worker") + try: + uploads = [traced_upload(user, chat["id"]) for _ in range(3)] + say(f"uploaded {len(uploads)} attachments: 202 queued (API accepts without worker)") + fired = {} + deadline = time.monotonic() + FIRE_TIMEOUT + while len(fired) < 2: + states = obs.alerts() + for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"): + if states.get(name) == "firing" and name not in fired: + fired[name] = round(time.monotonic() - stopped, 1) + say(f"ALERT {name} firing ({fired[name]} s after stop)") + if len(fired) < 2: + say(" consumers=%s backlog queued=%s oldest age=%s s, alerts: %s" % ( + q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'), + q('max(messenger_jobs_backlog{status="queued"})'), + rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1), + {k: v for k, v in states.items() if k.startswith("Messenger")})) + check(time.monotonic() < deadline, f"alerts not firing after {FIRE_TIMEOUT} s: {fired}") + time.sleep(5) + result["firing_after_stop_s"] = fired + + # Диагностика только по сохранённым наблюдениям: + diag = { + "queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'), + "broker_queue_depth": q('max(messenger_queue_depth{messaging_destination_name="attachments.process.v1"})'), + "jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'), + "oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1), + "outbox_oldest_age_s": rnd(q("max(messenger_outbox_oldest_age_seconds)"), 1), + "published_ok_last_5m": q('sum(increase(messenger_messages_published_total{outcome="ok"}[5m]))'), + "worker_metrics_last_seen_s": rnd(q('time() - max(timestamp(messenger_worker_capacity))'), 0), + } + last_worker_log = obs.loki_query('{service_name="messenger-worker"}', minutes=15, limit=1) + diag["last_worker_log"] = ({"event": last_worker_log[0][0].get("event"), "message": last_worker_log[0][1]} + if last_worker_log else None) + spans = obs.tempo_trace(uploads[0][1]) + names = sorted({s["name"] for s in spans}) + diag["upload_trace_span_names"] = names + diag["upload_trace_has_consumer_span"] = any(s["name"].startswith("process ") for s in spans) + result["diagnosis"] = diag + say("diagnosis: " + json.dumps(diag, ensure_ascii=False)) + check(diag["queue_consumers"] == 0, "consumers must be 0 while worker is stopped") + check("publish attachments" in names and not diag["upload_trace_has_consumer_span"], + "trace must end at publish: message is in the broker, nobody consumes it") + finally: + started = time.monotonic() + compose("start", "worker") + say("docker compose start worker") + ready_at = [] + for attachment_id, _ in uploads: + att = wait_attachment(user, attachment_id, seconds=120) + check(att["status"] == "ready", f"{attachment_id} is {att['status']}") + ready_at.append(round(time.monotonic() - started, 1)) + result["ready_after_start_s"] = ready_at + say(f"all attachments ready {max(ready_at)} s after worker start; backlog now %s" % + q('max(messenger_jobs_backlog{status="queued"})')) + resolved = {} + for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"): + obs.wait_alert(name, "resolved", RESOLVE_TIMEOUT, log=say) + resolved[name] = round(time.monotonic() - started, 1) + result["resolved_after_start_s"] = resolved + result["after_recovery"] = { + "jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'), + "oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1), + "queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})')} + say(f"alerts resolved: {result['resolved_after_start_s']}; now {result['after_recovery']}") + result["time_to_ready_p95_5m_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))'), 1) + save("worker-stop", result) + print("PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; " + "start -> backlog drained -> resolved") + + +# --------------------------------------------------------------------------------------------------- +LOCK_SECONDS = 210 # > окно rate (1m) + for (1m) + задержка экспорта метрик с запасом + + +def db_slow(): + """(2) PostgreSQL «медленный»: таблица messages заблокирована (ACCESS EXCLUSIVE) на LOCK_SECONDS. + Ожидаем MessengerHighErrorRate firing; трассы показывают, что время уходит в INSERT (PostgreSQL), + а не в обработчик; после снятия блокировки — resolved.""" + result = {"scenario": "db-slow", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "lock_seconds": LOCK_SECONDS} + load = Load(threads=8, think=0.05).start() + say("load: 8 threads POST /messages") + time.sleep(25) + base_from = time.monotonic() - 20 + result["baseline"] = load.window(base_from) + result["baseline"]["server_p95_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 4) + say(f"baseline: {result['baseline']}") + check(obs.alerts().get("MessengerHighErrorRate") is None, "MessengerHighErrorRate is already active") + + lock_out = {} + + def hold_lock(): + command = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", + "-q", "-c", f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({LOCK_SECONDS}); COMMIT;"] + lock_out["code"] = subprocess.run(command, cwd=ROOT, capture_output=True).returncode + lock_out["released"] = time.monotonic() # COMMIT выполнен: блокировка снята + + locker = threading.Thread(target=hold_lock, daemon=True) + locked = time.monotonic() + locker.start() + say(f"LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for {LOCK_SECONDS} s (PostgreSQL alive, writes wait)") + try: + fired_at = None + while fired_at is None: + if obs.alerts().get("MessengerHighErrorRate") == "firing": + fired_at = round(time.monotonic() - locked, 1) + break + say(" 5xx ratio=%s p95=%s s pool used=%s waiting=%s" % ( + rnd(q("messenger:http_error_ratio:rate1m"), 3), + rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'), 3), + q('sum(db_client_connection_count{db_client_connection_state="used"})'), + q("sum(db_client_connection_pending_requests)"))) + check(time.monotonic() - locked < LOCK_SECONDS + 30, "MessengerHighErrorRate did not fire") + time.sleep(5) + say(f"ALERT MessengerHighErrorRate firing ({fired_at} s after lock)") + result["firing_after_lock_s"] = fired_at + + # Диагностика: метрики + diag = { + "http_5xx_ratio_1m": rnd(q("messenger:http_error_ratio:rate1m"), 3), + "http_p95_messages_s": rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 3), + "pool_used": q('sum(db_client_connection_count{db_client_connection_state="used"})'), + "pool_wait_p95_s": rnd(q("histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))"), 3), + "status_codes_1m": {row[0].get("http_response_status_code"): rnd(row[1], 2) for row in obs.prom_query( + 'sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m]))')}, + } + # Чтения других таблиц во время блокировки messages: быстрые -> «медленно» не всё, а одна операция + reads = [] + for _ in range(5): + began = time.monotonic() + call("GET", "/api/v1/users/me", 200, token=load.user["token"]) + call("GET", "/api/v1/chats", 200, token=load.user["token"]) + reads.append((time.monotonic() - began) * 1000 / 2) + diag["reads_during_lock_ms"] = {"GET /users/me + GET /chats, avg per request": rnd(sum(reads) / len(reads), 1), + "max": rnd(max(reads), 1)} + # Логи: какие ошибки БД видит API + logs = obs.loki_query('{service_name="messenger-api"} | event="db.unavailable"', minutes=5, limit=500) + by_type = {} + for labels, _ in logs: + by_type[labels.get("error_type", "?")] = by_type.get(labels.get("error_type", "?"), 0) + 1 + diag["api_log_db_unavailable_by_error_type"] = by_type + # Трассы: медленный запрос с ошибкой — где время + found = obs.tempo_search('{ resource.service.name = "messenger-api" && name = "POST /api/v1/chats/{chat_id}/messages" && status = error }', + minutes=5, limit=5) + check(found, "no error traces of POST messages in Tempo") + spans = obs.wait_trace(found[0]["traceID"].rjust(32, "0"), seconds=30) + server = next(s for s in spans if s["kind"] == "SERVER") + db_spans = [s for s in spans if s["kind"] == "CLIENT" and s["name"] in ("INSERT", "SELECT", "UPDATE")] + slowest = max(db_spans, key=lambda s: s["duration_ms"]) + diag["error_trace"] = {"trace_id": found[0]["traceID"], "server_ms": rnd(server["duration_ms"], 1), + "slowest_db_span": slowest["name"], "slowest_db_ms": rnd(slowest["duration_ms"], 1), + "db_span_status": slowest["status"], "db_share": rnd(slowest["duration_ms"] / server["duration_ms"], 3), + "tree": obs.span_tree(spans).splitlines()} + # Контраст: медленный ОБРАБОТЧИК (CPU Argon2id при регистрации) — время в своём спане, БД быстрая + trace_id, traceparent = obs.new_traceparent() + call("POST", "/api/v1/users", 201, body={"username": f"cpu_{uuid.uuid4().hex[:10]}", "display_name": "Cpu", + "password": "Test-password-123"}, headers={"traceparent": traceparent}) + cpu = obs.wait_trace(trace_id, lambda s: any(x["name"] == "argon2id hash" for x in s), 30) + cpu_server = next(s for s in cpu if s["kind"] == "SERVER") + argon = next(s for s in cpu if s["name"] == "argon2id hash") + cpu_db = [s for s in cpu if s["kind"] == "CLIENT"] + diag["handler_cpu_contrast"] = {"trace_id": trace_id, "server_ms": rnd(cpu_server["duration_ms"], 1), + "argon2id_ms": rnd(argon["duration_ms"], 1), + "db_ms_total": rnd(sum(s["duration_ms"] for s in cpu_db), 1), + "tree": obs.span_tree(cpu).splitlines()} + result["diagnosis"] = diag + say("diagnosis: " + json.dumps({k: v for k, v in diag.items() if k not in ("error_trace", "handler_cpu_contrast")}, + ensure_ascii=False)) + say("error trace (DB): " + json.dumps({k: v for k, v in diag["error_trace"].items() if k != "tree"})) + print("\n".join(" " + line for line in diag["error_trace"]["tree"])) + say("contrast, slow handler (CPU): " + json.dumps({k: v for k, v in diag["handler_cpu_contrast"].items() if k != "tree"})) + print("\n".join(" " + line for line in diag["handler_cpu_contrast"]["tree"])) + check(diag["error_trace"]["db_share"] > 0.8, "in the DB incident most of the request time must be in a DB span") + check(diag["handler_cpu_contrast"]["argon2id_ms"] > diag["handler_cpu_contrast"]["db_ms_total"], + "in the CPU contrast the handler span must dominate the DB spans") + finally: + locker.join(timeout=LOCK_SECONDS + 30) + unlocked = lock_out.get("released", time.monotonic()) + say(f"lock released (psql exit {lock_out.get('code')}), {round(unlocked - locked, 1)} s after LOCK") + result["during_lock"] = load.window(locked + 5, unlocked) + obs.wait_alert("MessengerHighErrorRate", "resolved", RESOLVE_TIMEOUT, log=say) + result["resolved_after_unlock_s"] = round(time.monotonic() - unlocked, 1) + say(f"ALERT MessengerHighErrorRate resolved {result['resolved_after_unlock_s']} s after unlock") + result["after"] = load.window(unlocked + 5) + load.stop() + say(f"load: during lock {result['during_lock']}; after {result['after']}") + check((result["after"]["success_ratio"] or 0) > 0.99, "writes must recover after the lock") + save("db-slow", result) + print("PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span " + "(not the handler); unlock -> resolved") + + +# --------------------------------------------------------------------------------------------------- +def telemetry_down(stores=("tempo", "loki"), outage=150): + """(3) Хранилища трасс и логов недоступны дольше окна повторов Collector (120 с). + Ожидаем: API работает без ошибок, RSS Collector ограничен, TelemetryExportFailing firing, + потери считаются (send_failed) и видны на пробных трассах; после start — данные снова идут.""" + result = {"scenario": "telemetry-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "stopped": list(stores), + "outage_s": outage} + load = Load(threads=4, think=0.05).start() + time.sleep(20) + result["baseline"] = load.window(time.monotonic() - 15) + failed_before = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}") + for sig in ("spans", "log_records")} + heap_before = q("max(otelcol_process_runtime_heap_alloc_bytes)") + mem_before = container_mem_mib("otel-collector") + result["collector_heap_before_mib"] = rnd((heap_before or 0) / 2**20, 1) + result["collector_container_before_mib"] = mem_before + say(f"baseline: {result['baseline']}, collector heap {result['collector_heap_before_mib']} MiB, " + f"container {mem_before} MiB") + compose("stop", *stores) + stopped = time.monotonic() # отсчёт после остановки (Loki при stop сбрасывает данные на диск ~десятки секунд) + say(f"docker compose stop {' '.join(stores)}: stopped") + samples, probes = [], [] + try: + while time.monotonic() - stopped < outage: + probes.append((round(time.monotonic() - stopped, 1), *probe(load.user, load.chat["id"]))) + # memory_limiter ограничивает heap Go-процесса (RSS ещё включает отображённый в память бинарник ~150 MiB) + sample = {"t": round(time.monotonic() - stopped, 1), + "collector_heap_mib": rnd((q("max(otelcol_process_runtime_heap_alloc_bytes)") or 0) / 2**20, 1), + "collector_rss_mib": rnd((q("max(otelcol_process_memory_rss)") or 0) / 2**20, 1), + "collector_container_mib": container_mem_mib("otel-collector"), + "queue": {r[0].get("exporter"): r[1] for r in obs.prom_query("otelcol_exporter_queue_size")}, + "alert": obs.alerts().get("TelemetryExportFailing", "inactive")} + samples.append(sample) + say(f" {sample}") + time.sleep(10) + finally: + compose("start", *stores) + started = time.monotonic() + say(f"docker compose start {' '.join(stores)}") + result["during"] = load.window(stopped, started) + result["samples"] = samples + result["collector_heap_max_mib"] = max(s["collector_heap_mib"] or 0 for s in samples) + result["collector_rss_max_mib"] = max(s["collector_rss_mib"] or 0 for s in samples) + result["collector_container_max_mib"] = max(s["collector_container_mib"] or 0 for s in samples) + fired = [s["t"] for s in samples if s["alert"] == "firing"] + result["TelemetryExportFailing_firing_at_s"] = fired[0] if fired else None + # после восстановления: Collector досылает очередь; затем проверяем, какие пробные трассы дошли + obs.wait_metric("sum(otelcol_exporter_queue_size)", lambda v: v == 0, 180) + result["queue_drained_after_start_s"] = round(time.monotonic() - started, 1) + say(f"exporter queues drained {result['queue_drained_after_start_s']} s after start") + time.sleep(15) # Tempo/Loki индексируют принятые пакеты + found = [] + for t, trace_id, status in probes: + spans = [] + for _ in range(5): + spans = obs.tempo_trace(trace_id) + if spans: + break + time.sleep(2) + found.append({"sent_at_s": t, "http": status, "trace_in_tempo": bool(spans)}) + result["probe_traces"] = found + failed_after = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}") + for sig in ("spans", "log_records")} + result["lost_in_collector"] = {sig: round(failed_after[sig] - failed_before[sig]) for sig in failed_after} + obs.wait_alert("TelemetryExportFailing", "resolved", RESOLVE_TIMEOUT, log=say) + result["resolved_after_start_s"] = round(time.monotonic() - started, 1) + result["after"] = load.window(started + 5) + load.stop() + say(f"API during outage: {result['during']}") + say(f"probe traces: {sum(p['trace_in_tempo'] for p in found)}/{len(found)} reached Tempo " + f"(lost: sent at {[p['sent_at_s'] for p in found if not p['trace_in_tempo']]} s)") + say(f"lost in Collector (send_failed+enqueue_failed): {result['lost_in_collector']}; collector heap " + f"{result['collector_heap_before_mib']} -> max {result['collector_heap_max_mib']} MiB (memory_limiter 200 MiB), " + f"container max {result['collector_container_max_mib']} MiB") + check(result["during"]["success_ratio"] == 1.0, "API must keep working while telemetry stores are down") + check(result["collector_heap_max_mib"] < 200, "collector heap must stay below memory_limiter limit_mib") + save("telemetry-down", result) + print("PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved") + + +def collector_down(outage=90): + """(3') Сам Collector недоступен: SDK в процессах приложения ограничивает очередь и отбрасывает лишнее.""" + result = {"scenario": "collector-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "outage_s": outage} + load = Load(threads=4, think=0.05).start() + time.sleep(20) + result["baseline"] = load.window(time.monotonic() - 15) + # Потери SDK: отброшено процессором (очередь полна) + не принято exporter-ом (ошибка соединения). + sdk_lost = ('otel_sdk_processor_span_processed_total{job="messenger-api", error_type!=""}', + 'otel_sdk_exporter_span_exported_total{job="messenger-api", error_type!=""}') + lost_before = [counter(expr) for expr in sdk_lost] + api_before = container_mem_mib("api") + say(f"baseline: {result['baseline']}, api containers {api_before} MiB") + stopped = time.monotonic() + compose("stop", "otel-collector") + say("docker compose stop otel-collector") + samples = [] + try: + while time.monotonic() - stopped < outage: + samples.append({"t": round(time.monotonic() - stopped, 1), "api_mib": container_mem_mib("api")}) + say(f" {samples[-1]}") + time.sleep(10) + finally: + compose("start", "otel-collector") + started = time.monotonic() + say("docker compose start otel-collector") + result["during"] = load.window(stopped, started) + result["api_mib_before"] = api_before + result["api_mib_max"] = max(s["api_mib"] for s in samples) + time.sleep(30) # SDK досылает cumulative-метрики, включая счётчик отброшенных спанов + lost_after = [counter(expr) for expr in sdk_lost] + result["sdk_lost_spans_api"] = {"processor_queue_full": round(lost_after[0] - lost_before[0]), + "exporter_failed": round(lost_after[1] - lost_before[1])} + result["sdk_exported_ok_after_restart"] = round(q( + 'sum(increase(otel_sdk_exporter_span_exported_total{job="messenger-api", error_type=""}[1m]))', 0) or 0) + result["after"] = load.window(started + 5) + load.stop() + say(f"API during collector outage: {result['during']}; api memory {api_before} -> max {result['api_mib_max']} MiB; " + f"SDK lost spans (api): {result['sdk_lost_spans_api']}; exported OK in the last minute after restart: " + f"{result['sdk_exported_ok_after_restart']}") + check(sum(result["sdk_lost_spans_api"].values()) > 0, "losses during the outage must be counted by the SDK") + check(result["sdk_exported_ok_after_restart"] > 0, "spans must flow again after the Collector is back") + check(result["during"]["success_ratio"] == 1.0, "API must keep working without the Collector") + check(result["api_mib_max"] - api_before < 100, "API memory must not grow without bound") + save("collector-down", result) + print("PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start") + + +SCENARIOS = {"worker-stop": worker_stop, "db-slow": db_slow, "telemetry-down": telemetry_down, + "collector-down": collector_down} + +if __name__ == "__main__": + names = sys.argv[1:] or list(SCENARIOS) + unknown = [n for n in names if n not in SCENARIOS] + if unknown: + raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}") + failed = 0 + for name in names: + print(f"=== {name}", flush=True) + try: + SCENARIOS[name]() + except Exception as exc: # noqa: BLE001 + failed += 1 + print(f"FAIL: {name}: {exc}", flush=True) + for service in ("worker", "otel-collector", "tempo", "loki"): + compose("start", service) + print(f"{len(names) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..998548c --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/obs.py b/student-tests/obs.py new file mode 100644 index 0000000..4410e2c --- /dev/null +++ b/student-tests/obs.py @@ -0,0 +1,223 @@ +"""Запросы к хранилищам телеметрии для тестов lab6 (только стандартная библиотека). + +Адреса — порты, опубликованные compose.yaml только на 127.0.0.1: + PROMETHEUS_URL (http://127.0.0.1:9090), LOKI_URL (http://127.0.0.1:3100), TEMPO_URL (http://127.0.0.1:3200). +""" +import base64 +import binascii +import json +import os +import re +import secrets +import time +import urllib.error +import urllib.parse +import urllib.request + +PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090").rstrip("/") +LOKI_URL = os.environ.get("LOKI_URL", "http://127.0.0.1:3100").rstrip("/") +TEMPO_URL = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200").rstrip("/") + + +def get_json(url, timeout=10): + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=timeout) as reply: + return json.loads(reply.read().decode("utf-8")) + + +# ---------- Prometheus ---------- + +def prom_query(expr, at=None): + """Мгновенный запрос. Возвращает список (labels, float). Ошибка запроса -> исключение.""" + params = {"query": expr} + if at is not None: + params["time"] = str(at) + data = get_json(f"{PROMETHEUS_URL}/api/v1/query?" + urllib.parse.urlencode(params)) + if data.get("status") != "success": + raise RuntimeError(f"PromQL error: {data}") + result = data["data"]["result"] + if data["data"]["resultType"] == "scalar": + return [({}, float(result[1]))] + return [(item["metric"], float(item["value"][1])) for item in result] + + +def prom_value(expr, default=None): + rows = prom_query(expr) + return rows[0][1] if rows else default + + +def prom_series(match): + data = get_json(f"{PROMETHEUS_URL}/api/v1/series?" + urllib.parse.urlencode({"match[]": match})) + return data["data"] + + +def alerts(): + """Активные алерты: {имя: состояние (pending/firing)}.""" + data = get_json(f"{PROMETHEUS_URL}/api/v1/alerts") + out = {} + for alert in data["data"]["alerts"]: + name = alert["labels"]["alertname"] + if out.get(name) != "firing": + out[name] = alert["state"] + return out + + +def wait_alert(name, state, seconds, poll=5, log=print): + """Ждёт, пока алерт name перейдёт в state ('firing' или 'resolved' = отсутствует). Возвращает секунды.""" + started = time.monotonic() + while True: + current = alerts().get(name, "inactive") + done = current == "firing" if state == "firing" else current == "inactive" + if done: + return round(time.monotonic() - started, 1) + if time.monotonic() - started > seconds: + raise AssertionError(f"alert {name}: still {current} after {seconds} s (want {state})") + log(f" ... {name}: {current} ({time.monotonic() - started:.0f} s)") + time.sleep(poll) + + +def wait_metric(expr, predicate, seconds, poll=5): + """Ждёт, пока значение выражения удовлетворит predicate. Возвращает значение.""" + deadline = time.monotonic() + seconds + while True: + value = prom_value(expr) + if value is not None and predicate(value): + return value + if time.monotonic() > deadline: + raise AssertionError(f"{expr}: value {value} after {seconds} s") + time.sleep(poll) + + +# ---------- Loki ---------- + +def loki_query(logql, minutes=30, limit=200): + """Записи за последние minutes минут: список (labels+structured metadata, строка).""" + now = time.time_ns() + params = {"query": logql, "start": str(now - minutes * 60 * 10**9), "end": str(now), "limit": str(limit), + "direction": "backward"} + data = get_json(f"{LOKI_URL}/loki/api/v1/query_range?" + urllib.parse.urlencode(params)) + if data.get("status") != "success": + raise RuntimeError(f"LogQL error: {data}") + out = [] + for stream in data["data"]["result"]: + for value in stream["values"]: + labels = dict(stream["stream"]) + if len(value) > 2 and isinstance(value[2], dict): + for group in value[2].values(): + if isinstance(group, dict): + labels.update(group) + out.append((labels, value[1])) + return out + + +def wait_logs(logql, minimum=1, seconds=60): + deadline = time.monotonic() + seconds + while True: + rows = loki_query(logql) + if len(rows) >= minimum: + return rows + if time.monotonic() > deadline: + raise AssertionError(f"Loki: {len(rows)} records for {logql} after {seconds} s") + time.sleep(2) + + +# ---------- Tempo ---------- + +def new_traceparent(): + """Свой trace_id для контролируемого запроса: по нему трасса находится без поиска.""" + trace_id = secrets.token_hex(16) + return trace_id, f"00-{trace_id}-{secrets.token_hex(8)}-01" + + +def tempo_trace(trace_id): + """Спаны трассы: список dict(service, name, kind, span_id, parent_id, duration_ms, status, attrs, links).""" + try: + data = get_json(f"{TEMPO_URL}/api/traces/{trace_id}") + except urllib.error.HTTPError as error: + if error.code == 404: + return [] + raise + spans = [] + for batch in data.get("batches", data.get("resourceSpans", [])): + resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])} + for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])): + for span in scope.get("spans", []): + spans.append({ + "service": resource.get("service.name"), + "instance": resource.get("service.instance.id"), + "name": span["name"], + "kind": span.get("kind", "").replace("SPAN_KIND_", ""), + "span_id": _hex_id(span["spanId"], 16), + "parent_id": _hex_id(span.get("parentSpanId", ""), 16), + "start": int(span["startTimeUnixNano"]), + "duration_ms": (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6, + "status": span.get("status", {}).get("code", "").replace("STATUS_CODE_", ""), + "attrs": {a["key"]: _value(a["value"]) for a in span.get("attributes", [])}, + "links": [{"trace_id": _hex_id(link.get("traceId", ""), 32), + "span_id": _hex_id(link.get("spanId", ""), 16)} for link in span.get("links", [])], + }) + return spans + + +def _hex_id(value, size): + """ID спана/трассы из JSON Tempo: hex или base64 (OTLP JSON кодирует bytes в base64) -> hex.""" + if not value: + return "" + if re.fullmatch(r"[0-9a-fA-F]+", value) and len(value) in (size, size // 2 * 2): + return value.lower().rjust(size, "0") + try: + return base64.b64decode(value).hex().rjust(size, "0") + except (binascii.Error, ValueError): + return value + + +def _value(value): + for key in ("stringValue", "intValue", "boolValue", "doubleValue"): + if key in value: + return int(value[key]) if key == "intValue" else value[key] + return json.dumps(value) + + +def wait_trace(trace_id, predicate=lambda spans: bool(spans), seconds=60): + deadline = time.monotonic() + seconds + while True: + spans = tempo_trace(trace_id) + if predicate(spans): + return spans + if time.monotonic() > deadline: + raise AssertionError(f"Tempo: trace {trace_id} incomplete after {seconds} s ({len(spans)} spans)") + time.sleep(2) + + +def tempo_search(traceql, minutes=30, limit=20): + now = int(time.time()) + params = {"q": traceql, "start": str(now - minutes * 60), "end": str(now), "limit": str(limit)} + return get_json(f"{TEMPO_URL}/api/search?" + urllib.parse.urlencode(params)).get("traces", []) + + +def span_tree(spans): + """Текстовое дерево трассы для отчёта (без атрибутов с данными пользователя).""" + by_id = {s["span_id"]: s for s in spans} + children = {} + for span in spans: + parent = span["parent_id"] if span["parent_id"] in by_id else "" + children.setdefault(parent, []).append(span) + lines = [] + + def walk(span, depth): + extra = [] + for key in ("messenger.job.attempt", "messenger.job.outcome", "http.response.status_code", "error.type", + "aws.s3.key"): + if key in span["attrs"]: + extra.append(f"{key}={span['attrs'][key]}") + status = " ERROR" if span["status"] == "ERROR" else "" + links = f" links={len(span['links'])}" if span["links"] else "" + short = (span["service"] or "?").replace("messenger-", "") + lines.append(f"{' ' * depth}- [{short}] {span['name']} ({span['kind']}, {span['duration_ms']:.1f} ms)" + f"{status}{links} {' '.join(extra)}".rstrip()) + for child in sorted(children.get(span["span_id"], []), key=lambda s: s["start"]): + walk(child, depth + 1) + + for root in sorted(children.get("", []), key=lambda s: s["start"]): + walk(root, 0) + return "\n".join(lines) diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 0000000..6535eae --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 0000000..f2f3ffb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 0000000..0f28ec8 --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 0000000..80c555d --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,51 @@ +import os +import re +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + match = re.match(r"^.+?-lab\d+-(.+)-\d+$", name) + restarted.append(match.group(1) if match else name) + services = ("postgres", "redis", "s3", "rabbitmq", "api", "janitor", "dispatcher", "worker", "lb", + "otel-collector", "prometheus", "loki", "tempo", "grafana") + for service in services: + check(service in restarted, f"{service} was not restarted by the hook") + print(f"PASS: the hook restarted {', '.join(services)} (all api replicas; same containers, new start time); " + f"data, the attachment bytes and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..7c770d9 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,258 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_empty_cursor, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..e57fe28 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,338 @@ +"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ. + +Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа: +пользовательская ошибка не должна становиться 500. +""" +import hashlib +import http.client +import socket +import struct +import urllib.parse +import uuid +import zlib + +import images +from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload + +URL = urllib.parse.urlsplit(BASE_URL) + + +def setup(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + return alice, bob, eve, new_chat(alice, bob) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def check_error(reply, code): + check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}") + + +def test_png_and_jpeg_roundtrip(): + alice, bob, _, chat = setup() + for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"), + (images.sample_jpeg(), "image/jpeg", "sample.jpg"), + (images.progressive_jpeg(), "image/jpeg", "progressive.jpg"), + (images.png(33, 17, interlace=1), "image/png", "interlaced.png"), + (images.png(1, 1, gray=True), "image/png", "1x1.png")): + att = upload(alice, chat["id"], data, ctype, name).json + check(att["status"] == "ready" and att["error_code"] is None, "not ready") + check([v["name"] for v in att["variants"]] == ["thumbnail", "crop", "optimized"], "lab5 variants missing") + check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ") + check(att["content_type"] == ctype and att["filename"] == name, "metadata differs") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta == att, "GET metadata differs from POST response") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "downloaded bytes differ") + check(got.headers["content-type"] == ctype, "wrong Content-Type on download") + check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length") + check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing") + print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes") + print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte") + + +def test_size_limit_exact_boundary(): + alice, _, _, chat = setup() + exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES) + check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB") + att = upload(alice, chat["id"], exact).json + check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed") + print(f"exactly {len(exact)} bytes (10 MiB) -> 202, then ready") + over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1) + reply = upload(alice, chat["id"], over, expected=413) + check_error(reply, "payload_too_large") + print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body") + print("PASS: the 10 MiB limit is exact") + + +def chunked_upload(user, chat_id, total, piece=64 * 1024): + """Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении.""" + conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60) + + def body(): + sent = 0 + head = images.sample_png()[:64] + yield head + sent += len(head) + while sent < total: + part = bytes(min(piece, total - sent)) + sent += len(part) + yield part + try: + conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(), + headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]}, + encode_chunked=True) + reply = conn.getresponse() + return reply.status, reply.read() + finally: + conn.close() + + +def test_size_limit_streaming(): + alice, _, _, chat = setup() + status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024) + check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}") + print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed") + call("GET", "/health/ready", 200) + print("PASS: oversized streamed body is cut off, the API keeps working") + + +def test_pixel_limit(): + alice, _, _, chat = setup() + at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей + att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json + print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 202, ready {att['id'][:8]}") + over = images.png(8001, 5000, gray=True) + check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large") + print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413") + bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01") + check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large") + print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded") + print("PASS: pixel limit is checked before decompression") + + +def test_damaged_images(): + alice, _, _, chat = setup() + png, jpeg = images.sample_png(), images.sample_jpeg() + cases = [ + ("PNG cut in half", png[:len(png) // 2], "image/png"), + ("PNG without IEND", png[:-12], "image/png"), + ("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"), + ("PNG with garbage after IEND", png + b"garbage", "image/png"), + ("PNG signature only", png[:8], "image/png"), + ("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"), + ("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"), + ("JPEG header only", jpeg[:20], "image/jpeg"), + ("random bytes", uuid.uuid4().bytes * 100, "image/png"), + ("plain text", "это не картинка".encode(), "image/jpeg"), + ] + for name, data, ctype in cases: + reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415) + check_error(reply, "unsupported_media_type") + print(f"{name} ({len(data)} bytes, {ctype}) -> 415") + check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request") + print("empty body -> 400") + print("PASS: damaged and foreign files are rejected with 415, never 500") + + +def test_png_chunk_semantics(): + """CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201. + + Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready, + хотя Pillow его отклоняет (invalid palette size). + """ + alice, bob, _, chat = setup() + from images import assemble, ihdr, zero_idat + pal = lambda entries: (b"PLTE", bytes(3 * entries)) + idat_p = zero_idat(1, 1, 8, 1) + bad = [ + ("PLTE 771 bytes = 257 entries (review case)", + assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))), + ("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)), + ("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)), + ("palette larger than bit depth allows (depth 1, 3 entries)", + assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))), + ("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)), + ("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))), + ("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))), + ("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)), + ("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)), + ("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)), + ("tRNS of wrong size for RGB (4 bytes instead of 6)", + assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))), + ("tRNS in an image with alpha channel", + assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))), + ("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))), + ("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))), + ("IDAT chunks not consecutive", + assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"), + (b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))), + ("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))), + ("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))), + ("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)), + zero_idat(1, 1, 8, 3))), + ("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))), + ("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))), + ("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))), + ("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))), + ("zTXt with broken zlib after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))), + ("zTXt text bomb: 50 MB of text in ~50 KB after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))), + ] + for name, data in bad: + check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type") + print(f"{name} ({len(data)} bytes, valid CRCs) -> 415") + good = [ + ("palette 256 entries + tRNS + tEXt and tIME after IDAT", + assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1), + (b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))), + ("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))), + ("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)), + (b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))), + ("compressed zTXt (100 KB of text) and iTXt after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)), + (b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))), + ] + for name, data in good: + att = upload(alice, chat["id"], data, filename="good.png").json + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "valid PNG changed") + print(f"{name} -> 201, downloaded byte for byte") + print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted") + + +def test_mime_substitution(): + alice, _, _, chat = setup() + cases = [ + ("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"), + ("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"), + ("GIF bytes sent as image/png", images.GIF, "image/png"), + ("PNG sent as image/gif", images.sample_png(), "image/gif"), + ("PNG sent as text/html", images.sample_png(), "text/html"), + ("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"), + ("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"), + ] + for name, data, ctype in cases: + check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type") + print(f"{name} -> 415") + att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json + check(att["content_type"] == "image/png", "content type must be normalised") + print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png") + print("PASS: the type is taken from the content; a declared type that does not match is refused") + + +def test_filenames(): + alice, _, _, chat = setup() + bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png", + "bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256] + for name in bad: + check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request") + print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400") + check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request") + print("no filename -> 400") + for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"): + att = upload(alice, chat["id"], images.sample_png(), filename=name).json + check(att["filename"] == name, "display name changed") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded") + print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only") + print("PASS: filename never becomes a path; dangerous names are refused") + + +def test_access_control(): + alice, bob, eve, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + for suffix in ("", "/content"): + reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + check_error(reply, "not_found") + call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400") + check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found") + check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found") + upload(None, chat["id"], images.sample_png(), expected=401) + print("upload into someone else's / unknown chat -> 404, without token -> 401") + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=preview", 400, token=bob["token"]) + for variant in ("original", "thumbnail", "crop", "optimized"): + call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}", 200, token=bob["token"]) + print("variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)") + print("PASS: only chat members can see an attachment") + + +def test_message_attachments(): + alice, bob, eve, chat = setup() + first = upload(alice, chat["id"], images.sample_png()).json + second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json + msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json + check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed") + print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member") + plain = post_message(bob, chat["id"], "без вложений").json + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json + history, _ = read_history(bob, chat["id"]) + check(history == [msg, plain, reused], "history differs from the confirmed messages") + print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed") + + other_chat = new_chat(alice, eve) + foreign = upload(alice, other_chat["id"], images.sample_png()).json + for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"), + ([first["id"], foreign["id"]], "own + foreign attachment"), + ([str(uuid.uuid4())], "unknown attachment_id")): + check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found") + print(f"{what} -> 404") + check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found") + print("outsider referencing an attachment of the chat -> 404") + for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"), + ([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"), + (["not-a-uuid"], "x", "bad UUID"), + ([], "", "empty text without attachments"), + ([], " ", "whitespace text without attachments")): + check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request") + print(f"{what} -> 400") + history_after, _ = read_history(bob, chat["id"]) + check(history_after == history, "a rejected message was stored") + print("PASS: only attachments of the same chat can be linked; rejected requests change nothing") + + +def test_idempotency_with_attachments(): + alice, bob, _, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json + key = new_key() + first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + check(again == first, "replay returned a different message") + check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict") + check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict") + key2 = new_key() + plain = post_message(alice, chat["id"], "текст", None, key=key2).json + check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field") + history, _ = read_history(bob, chat["id"]) + check(len(history) == 2, "replays created extra messages") + print("same key + same attachments -> same 201; other attachments -> 409; missing field == []") + print("PASS: Idempotency-Key covers attachment_ids") + + +def test_aborted_upload(): + alice, _, _, chat = setup() + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000)) + sock.close() # клиент пропал на середине тела + print("sent 1.1 MB of a declared 5 MB body and closed the connection") + att = upload(alice, chat["id"], images.sample_png()).json + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + print("the API keeps serving uploads; nothing of the aborted request is visible " + "(DB/S3 side is checked by test_consistency.py)") + print("PASS: an interrupted upload is dropped") + + +if __name__ == "__main__": + run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, + test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control, + test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..01f2251 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,103 @@ +from collections import Counter + +import images +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_attachment_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + sent = upload(alice, chat["id"], data) + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"]) + check(reply.data == data, "another replica returned different bytes") + readers.add(reply.instance) + print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}") + check(readers - {sent.instance}, "no download went to another replica") + print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_attachment_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 0000000..ff3d4ab --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,276 @@ +"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка). + +Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`), +выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает +janitor вручную. Данные синтетические, volumes не удаляются. + + python student-tests/test_consistency.py [s3|crash|orphan|aborted|all] +""" +import json +import socket +import sys +import time +import urllib.parse + +import images +from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat, + new_user, post_message, run_tests, sql_value, upload, wait_ready) + +URL = urllib.parse.urlsplit(BASE_URL) + +# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но enqueue_upload +# (lab5: UPDATE ... state='queued' + задание + outbox, COMMIT) падает — как если бы реплика умерла +# сразу после PUT в staging, до фиксации задания. +CRASH_AFTER_PUT = r''' +import json, struct, sys, tempfile, zlib +from uuid import UUID +import attachments, db, storage, uploads +chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2]) +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b"")) +f = tempfile.SpooledTemporaryFile(); f.write(data) +import hashlib +upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest()) +image = uploads.inspect(upload, "image/png") +created = {} +real_intent = db.create_upload_intent +def remember(attachment_id, *args): + created["id"], created["key"] = str(attachment_id), args[-1] + return real_intent(attachment_id, *args) +def crash(attachment_id, *args): + raise RuntimeError("simulated crash between S3 PUT and DB COMMIT") +db.create_upload_intent = remember +db.enqueue_upload = crash +db.pool.open() +try: + attachments.store(chat_id, owner_id, "crash.png", upload, image) + outcome = "stored" +except RuntimeError as exc: + outcome = str(exc) +print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"], + "object_exists": storage.exists(created["key"])})) +''' + +OBJECT_EXISTS = r''' +import json, sys, storage +print(json.dumps({"exists": storage.exists(sys.argv[1])})) +''' + +PUT_ORPHAN = r''' +import hashlib, io, json, uuid, storage +data = b"orphan object without a row in PostgreSQL" +key = storage.new_key(uuid.uuid4().hex) +storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest()) +print(json.dumps({"key": key})) +''' + +DELETE_OBJECT = r''' +import json, sys, storage +storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]})) +''' + +COUNT_OBJECTS = r''' +import json, storage +print(json.dumps({"objects": sum(1 for prefix in storage.ALL_PREFIXES for _ in storage.list_objects(prefix))})) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def object_exists(key): + return last_json(app_python(OBJECT_EXISTS, key))["exists"] + + +def backdate(where): + sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}") + + +def uploading_rows(chat_id): + return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'")) + + +def wait_download(user, att, seconds=60): + deadline = time.monotonic() + seconds + while True: + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"]) + if reply.status == 200: + return reply + check(time.monotonic() < deadline, "download did not recover after S3 start") + time.sleep(1) + + +def s3_outage(stop_action, start_action, label): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + att = upload(alice, chat["id"], data).json + before = uploading_rows(chat["id"]) + print(f"--- S3 {label}: docker compose {stop_action} s3") + compose(stop_action, "s3") + try: + started = time.monotonic() + reply = upload(alice, chat["id"], data, expected=503) + ms = (time.monotonic() - started) * 1000 + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + body = reply.data.decode().lower() + check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")), + "error leaks storage details") + print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body") + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + print("download of an existing file -> 503") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta["sha256"] == att["sha256"], "metadata changed") + post_message(alice, chat["id"], "текст без файла") + post_message(alice, chat["id"], "", [att["id"]]) + call("GET", "/health/ready", 200) + print("metadata, messages (also with an already stored attachment) and /health/ready keep working: " + "S3 is needed only for file bytes") + after = uploading_rows(chat["id"]) + print(f"uploading rows of this chat: before {before}, after the failed upload {after} " + "(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API") + finally: + print(f"docker compose {start_action} s3") + compose(start_action, "s3") + got = wait_download(bob, att) + check(got.data == data, "file changed after S3 recovery") + print("after S3 recovery the old file downloads byte for byte") + upload(alice, chat["id"], data) + print("new upload -> 201") + leftover = uploading_rows(chat["id"]) + if leftover: + backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'") + report = janitor() + print(f"janitor after grace period: {json.dumps(report)}") + check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain") + print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up") + + +def test_s3_stopped(): + s3_outage("stop", "start", "stopped (connection refused / name not resolved)") + + +def test_s3_frozen(): + s3_outage("pause", "unpause", "frozen (no answer, timeouts)") + + +def test_crash_between_s3_and_db(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"])) + check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}") + att_id, key = result["attachment_id"], result["object_key"] + state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'") + check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3") + print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3") + call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"]) + post_message(alice, chat["id"], "x", [att_id], expected=404) + print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404") + + report = janitor() + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key), + "fresh upload was deleted inside the grace period") + print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> " + "fresh row and object are kept (the upload might still be running)") + backdate(f"id = '{att_id}'") + report = janitor("--dry-run") + check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row") + print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted") + report = janitor() + check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists") + check(not object_exists(key), "object still exists") + print(f"janitor: {json.dumps(report)} -> row and object deleted") + report = janitor() + print(f"janitor again: {json.dumps(report)} -> idempotent") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment damaged by cleanup") + print("the ready attachment of the same chat is untouched") + print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period") + + +def test_orphan_objects(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + key = last_json(app_python(PUT_ORPHAN))["key"] + print(f"object {key} put into the bucket without a row in PostgreSQL") + # Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с. + short = {"UPLOAD_TIMEOUT_SECONDS": "1"} + report = janitor("--grace-seconds", "3", env=short) + check(object_exists(key), "fresh orphan deleted inside grace period") + print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept") + time.sleep(4) + report = janitor("--grace-seconds", "3", "--dry-run", env=short) + check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key), + "dry run must report and keep the orphan") + print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept") + report = janitor("--grace-seconds", "3", env=short) + check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted") + print(f"janitor: {json.dumps(report)} -> orphan deleted") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment was deleted with orphans") + print("ready attachments older than the grace period are untouched (they have rows)") + + victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json + victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'") + app_python(DELETE_OBJECT, victim_key) + report = janitor("--grace-seconds", "3", env=short) + check(report["missing_objects"] >= 1, "missing object not reported") + check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted") + reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"]) + print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects=" + f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)") + print("PASS: orphans are found and deleted only after the grace period; missing objects are reported") + + +def api_tmp_files(): + counts = {} + for container in compose_output("ps", "-q", "api").split(): + counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0) + return counts + + +def test_aborted_upload_leaves_nothing(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + objects = last_json(app_python(COUNT_OBJECTS))["objects"] + rows = int(sql_value("SELECT count(*) FROM attachments")) + for sent in (100_000, 3_000_000): + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n" + f"Content-Length: 9000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(sent)) + sock.close() + print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection") + time.sleep(2) + check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row") + check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object") + print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}") + tmp = api_tmp_files() + check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}") + print(f"temporary upload files in api containers: {tmp} (removed when the request ends)") + for container in compose_output("ps", "-q", "api").split(): + mounts = docker("inspect", "-f", "{{json .Mounts}}", container) + # файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных + storage = [m for m in json.loads(mounts) + if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)] + check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}") + print("api containers have no volumes: the original is never stored locally") + print("PASS: an interrupted upload leaves no row, no object and no local file") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db], + "orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]} + groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"] + check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]") + wait_ready() + run_tests(groups[target]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 0000000..3d71fcc --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..e6017ce --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 0000000..8eaef38 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..407dea3 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_janitor_replay.py b/student-tests/test_janitor_replay.py new file mode 100644 index 0000000..4706a93 --- /dev/null +++ b/student-tests/test_janitor_replay.py @@ -0,0 +1,154 @@ +"""Регрессия: janitor не должен удалять объекты, которые replay записывает заново. + +Дефект (найден на ревью): у failed-вложения остаются объекты под attachments//v1/ (например, worker упал +после PUT, затем попытки исчерпаны). Janitor считал их сиротами, хотя failed ещё можно повторить (исходник +в staging хранится STAGING_RETENTION_SECONDS). Повторная проверка и DELETE не были синхронизированы с replay: +janitor, приостановленный после проверки, удалял crop.jpg, только что записанный успешным replay, +а метаданные оставались ready -> /content?variant=crop -> 503. + +Сценарий воспроизводит ровно это чередование: janitor запускается в контейнере janitor с перехватом +storage.delete — перед удалением любого объекта этого вложения он печатает PAUSED и ждёт; в это время тест +делает штатный ops.py replay и ждёт ready, потом отпускает janitor. Ожидание после исправления: janitor +вообще не пытается удалить объекты failed-вложения, пока replay возможен; после replay все 4 файла — 200. +Вторая часть: когда исходник удалён по retention (replay больше невозможен — состояние конечное), +оставшиеся объекты failed-вложения удаляются, а replay отказывает. + + python student-tests/test_janitor_replay.py +Длительность ~4 мин (объекты должны стать старше grace = 120 с, минимально допустимого). +""" +import json +import subprocess +import threading +import time + +from common import ROOT, call, check, run_tests, wait_ready +from test_jobs import (clear_faults, content, fault, get, image, log, ops, probe, upload, user_and_chat, + wait_status, job) + +GRACE = 120 # минимально допустимый grace: 2 x UPLOAD_TIMEOUT_SECONDS + +HOOK = r''' +import json, pathlib, sys, time +import janitor, storage +attachment_id, grace = sys.argv[1], int(sys.argv[2]) +prefix = f"attachments/{attachment_id}/" +release = pathlib.Path("/tmp/janitor-release-" + attachment_id) +real_delete = storage.delete +deleted = [] +def delete(key, *args, **kwargs): + if key.startswith(prefix): + print("PAUSED " + key, flush=True) # проверка уже пройдена, DELETE ещё не выполнен + deadline = time.time() + 120 + while not release.exists() and time.time() < deadline: + time.sleep(0.2) + deleted.append(key) + return real_delete(key, *args, **kwargs) +storage.delete = delete +report = None +for _ in range(60): # фоновый janitor может держать advisory lock + report = janitor.run_once(grace, False) + if report is not None: + break + time.sleep(2) +print(json.dumps({"report": report, "deleted_ours": deleted}), flush=True) +''' + + +def exec_python(service, code, *args, env=None): + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode(), capture_output=True, check=False) + check(result.returncode == 0, f"{service}: {result.stderr.decode('utf-8', 'replace')[-500:]}") + return result.stdout.decode("utf-8", "replace") + + +def failed_with_leftovers(name): + """failed/retry_exhausted, у которого под attachments//v1/ остались объекты прерванной попытки.""" + clear_faults() + fault("worker_crash_after_put") # попытка 1: объекты записаны, worker убит до COMMIT + att = upload(image("thumb320"), filename=f"{name}.png") + deadline = time.monotonic() + 30 + while len(probe("inspect", att["id"])["final_keys"]) < 4: + check(time.monotonic() < deadline, "attempt 1 did not write its objects") + time.sleep(0.3) + written = time.monotonic() + fault("worker_s3_error", 4) # попытки 2-5: временная ошибка S3 -> retry_exhausted + failed = wait_status(att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", f"{name}: {failed}") + keys = probe("inspect", att["id"])["final_keys"] + check(len(keys) == 4, f"{name}: leftovers expected, got {keys}") + log(f"{name}: failed/retry_exhausted after a crash post-PUT; leftover objects {keys}") + return att, written + + +def test_janitor_does_not_delete_replayed_objects(): + user_and_chat() + a, written_a = failed_with_leftovers("race-replayable") + b, written_b = failed_with_leftovers("race-terminal") + wait = max(written_a, written_b) + GRACE + 5 - time.monotonic() # объекты ОБОИХ старше grace + if wait > 0: + log(f"waiting {wait:.0f} s until the leftovers are older than grace ({GRACE} s)") + time.sleep(wait) + + # 1. janitor проходит bucket; если решит удалить объект A — остановится перед DELETE + proc = subprocess.Popen(["docker", "compose", "exec", "-T", "janitor", "python", "-", a["id"], str(GRACE)], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + proc.stdin.write(HOOK.encode()) + proc.stdin.close() + lines = [] + reader = threading.Thread(target=lambda: lines.extend(l.decode().strip() for l in proc.stdout), daemon=True) + reader.start() + deadline = time.monotonic() + 60 + while not any(l.startswith("PAUSED") for l in lines) and proc.poll() is None: + check(time.monotonic() < deadline, "janitor pass did not finish") + time.sleep(0.2) + paused = [l for l in lines if l.startswith("PAUSED")] + log(f"janitor pass over the bucket: {'paused before DELETE of ' + paused[0][7:] if paused else 'did not try to delete any object of the replayable failed attachment'}") + + # 2. штатный replay, пока janitor стоит (или уже прошёл) + j = job(a["id"]) + check("replayed" in ops("replay", j["job_id"], "--reason", "janitor race regression"), "replay refused") + wait_status(a["id"], "ready", 60) + log(f"ops replay -> job {j['job_id']} ready") + + # 3. отпускаем janitor и ждём конца прохода + exec_python("janitor", "import pathlib, sys\npathlib.Path('/tmp/janitor-release-' + sys.argv[1]).touch()\n", a["id"]) + proc.wait(timeout=120) + reader.join(timeout=5) + result = json.loads([l for l in lines if l.startswith("{")][-1]) + check(result["report"] is not None, "janitor pass did not run") + + # 4. результат replay цел + for variant in ("original", "thumbnail", "crop", "optimized"): + content(a["id"], variant, expected=200) + info = probe("inspect", a["id"]) + check(info["final_keys"] == ["v1/crop.jpg", "v1/optimized.webp", "v1/original", "v1/thumbnail.jpg"], + f"objects after the janitor pass: {info['final_keys']}") + check(not result["deleted_ours"], f"janitor deleted objects of the replayed attachment: {result['deleted_ours']}") + log("after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; " + f"janitor report: orphans {result['report']['orphan_objects']}, deleted {result['report']['orphan_objects_deleted']}") + + # 5. B всё ещё можно повторить -> его остатки тоже не тронуты + check(len(probe("inspect", b["id"])["final_keys"]) == 4, "leftovers of a replayable failed attachment deleted") + + # 6. retention истёк -> исходника нет, replay невозможен (конечное состояние) -> остатки удаляются + terminal = exec_python("janitor", "import janitor, json, sys, time\n" + "for _ in range(60):\n" + " r = janitor.run_once(int(sys.argv[1]), False)\n" + " if r is not None:\n break\n time.sleep(2)\n" + "print(json.dumps(r))\n", GRACE, env={"STAGING_RETENTION_SECONDS": "1"}) + left = probe("inspect", b["id"])["final_keys"] + check(left == [], f"leftovers of a terminal failed attachment must be swept, got {left}; janitor: " + f"{terminal.strip().splitlines()[-1]}") + refused = ops("replay", job(b["id"])["job_id"]) + check("refused" in refused and "staging" in refused["refused"], f"replay must be refused: {refused}") + check(get(b["id"])["status"] == "failed", "terminal attachment must stay failed") + log(f"retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of " + f"the failed attachment swept; replay refused: \"{refused['refused']}\"") + + +if __name__ == "__main__": + wait_ready() + run_tests([test_janitor_does_not_delete_replayed_objects]) diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 0000000..b382ee2 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,567 @@ +"""Lab5: асинхронная обработка вложений — варианты, сохранность, отказы, повторы, DLQ, replay. + +Нужен запущенный стенд из compose.yaml с FAULT_INJECTION=1 (по умолчанию на учебном стенде). +Тест управляет только своими синтетическими данными и сервисами через `docker compose stop|start|restart`, +volumes не удаляет. Сбои внедряются файлами в volume faults (faults.py), SQL — только чтение. + + python student-tests/test_jobs.py [сценарий ...] # без аргументов — все по порядку + +Сценарии: variants limits worker-down broker-down outbox-crash crash-after-put crash-after-commit + lease retry exhausted-replay permanent concurrency dlq +""" +import base64 +import json +import sys +import time +import urllib.parse +from pathlib import Path + +from common import (call, check, compose, compose_output, new_chat, new_user, post_message, psql, run_tests, + send, sql_value, wait_ready) + +HERE = Path(__file__).resolve().parent +PROBE = (HERE / "worker_probe.py").read_text(encoding="utf-8") +FINAL_KEYS = ["v1/crop.jpg", "v1/optimized.webp", "v1/original", "v1/thumbnail.jpg"] +LEASE = 15 # JOB_LEASE_SECONDS стенда +TIMEOUT = 60 # JOB_TIMEOUT_SECONDS стенда +MAX_ATTEMPTS = 5 # JOB_MAX_ATTEMPTS стенда + +STATE = {} + + +# ---------- инфраструктура ---------- + +def in_container(service, code, *args): + """python - в контейнере service, код через stdin. Последняя строка stdout — JSON. + Если service остановлен (сценарии с выключенным worker) — в одноразовом контейнере того же сервиса.""" + import subprocess + from common import ROOT + running = subprocess.run(["docker", "compose", "ps", "--status", "running", "-q", service], cwd=ROOT, + capture_output=True, check=False).stdout.strip() + launch = (["exec", "-T", service, "python"] if running + else ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service]) + result = subprocess.run(["docker", "compose", *launch, "-", *map(str, args)], cwd=ROOT, + input=code.encode("utf-8"), capture_output=True, check=False) + check(result.returncode == 0, f"{service}: {result.stderr.decode('utf-8', 'replace')[-500:]}") + return json.loads(result.stdout.decode("utf-8").strip().splitlines()[-1]) + + +def probe(*args): + return in_container("worker", PROBE, *args) + + +def ops(*args): + import subprocess + from common import ROOT + result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], + cwd=ROOT, capture_output=True, check=False) + check(result.returncode == 0, f"ops {args[0]}: {result.stderr.decode('utf-8', 'replace')[-400:]}") + return json.loads(result.stdout.decode("utf-8")) + + +def fault(point, times=1): + """Взвести точку сбоя: times файлов /faults/. (каждое срабатывание удаляет один файл).""" + code = ("import pathlib, sys, time\n" + "for n in range(int(sys.argv[2])):\n" + " pathlib.Path('/faults', f'{sys.argv[1]}.{time.time_ns()}_{n}').touch()\n" + "print('{}')\n") + in_container("worker", code, point, times) + + +def clear_faults(): + code = "import pathlib\nfor p in pathlib.Path('/faults').glob('*'):\n p.unlink()\nprint('{}')\n" + in_container("worker", code) + + +def image(kind): + return base64.b64decode(probe("make", kind)["b64"]) + + +def user_and_chat(): + if "alice" not in STATE: + STATE["alice"] = new_user("alice") + STATE["bob"] = new_user("bob") + STATE["chat"] = new_chat(STATE["alice"], STATE["bob"], title="Lab5 jobs") + return STATE["alice"], STATE["bob"], STATE["chat"] + + +def upload(data, content_type="image/png", filename="probe.png", expected=202): + alice, _bob, chat = user_and_chat() + query = "?filename=" + urllib.parse.quote(filename, safe="") + att = call("POST", f"/api/v1/chats/{chat['id']}/attachments{query}", expected, raw=data, token=alice["token"], + headers={"Content-Type": content_type}, timeout=60).json + if expected == 202: + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, + f"202 must be a queued snapshot, got {att['status']}") + return att + + +def get(att_id, user=None): + user = user or STATE["alice"] + return call("GET", f"/api/v1/attachments/{att_id}", 200, token=user["token"]).json + + +def wait_status(att_id, statuses, seconds=60): + statuses = (statuses,) if isinstance(statuses, str) else statuses + deadline = time.monotonic() + seconds + while True: + att = get(att_id) + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"{att_id[:8]}: still {att['status']} after {seconds} s") + time.sleep(0.3) + + +def content(att_id, variant="original", expected=200): + return call("GET", f"/api/v1/attachments/{att_id}/content?variant={variant}", expected, + token=STATE["bob"]["token"]) + + +def job(att_id): + rows = sql_value(f"SELECT job_id || '|' || status || '|' || attempt || '|' || coalesce(error_code, '-') " + f"FROM attachment_jobs WHERE attachment_id = '{att_id}'") + check(rows and "\n" not in rows, f"expected exactly one job for {att_id}") + job_id, status, attempt, error = rows.split("|") + return {"job_id": job_id, "status": status, "attempt": int(attempt), "error_code": error} + + +def events(job_id): + out = sql_value(f"SELECT attempt || ':' || event || ':' || coalesce(detail, '') FROM job_events " + f"WHERE job_id = '{job_id}' ORDER BY id") + return out.splitlines() if out else [] + + +def names(evs): + return [e.split(":")[1] for e in evs] + + +def assert_one_result(att_id, label=""): + """Инвариант идемпотентности: ровно один набор вариантов и ровно 4 объекта под конечным prefix.""" + info = probe("inspect", att_id) + check(info["state"] == "ready", f"{label}: not ready in DB") + check(info["variant_rows"] == 3, f"{label}: {info['variant_rows']} variant rows, expected 3") + check(info["final_keys"] == FINAL_KEYS, f"{label}: final objects {info['final_keys']}") + check(info["original_sha_ok"] and info["original_size_ok"], f"{label}: original differs from upload") + check(all(v["sha_ok"] and v["meta_size_ok"] for v in info["variants"].values()), f"{label}: variant mismatch") + return info + + +def outbox_rows(job_id): + out = sql_value(f"SELECT attempt || ':' || (published_at IS NOT NULL) FROM outbox WHERE job_id = '{job_id}' " + f"ORDER BY id") + return out.splitlines() if out else [] + + +def queue_depth(): + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_queues", "-p", "messenger", + "name", "messages", "messages_unacknowledged") + depth = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[1].isdigit(): + depth[parts[0]] = (int(parts[1]), int(parts[2])) + return depth + + +def log(text): + print(text, flush=True) + + +# ---------- сценарии ---------- + +def test_variants(): + """thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.""" + alice, bob, chat = user_and_chat() + cases = { + "thumb320": ("image/png", {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [320, 200]}), + "exif6": ("image/jpeg", {"thumbnail": [192, 256], "crop": [128, 128], "optimized": [300, 400]}), + "alpha": ("image/png", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [300, 200]}), + "tiny": ("image/png", {"thumbnail": [40, 30], "crop": [128, 128], "optimized": [40, 30]}), + "large": ("image/jpeg", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [1280, 853]}), + "strip": ("image/png", {"thumbnail": [256, 1], "crop": [128, 128], "optimized": [1280, 1]}), + } + for kind, (ctype, sizes) in cases.items(): + data = image(kind) + att = upload(data, ctype, f"{kind}.{'jpg' if ctype == 'image/jpeg' else 'png'}") + ready = wait_status(att["id"], "ready") + meta = {v["name"]: v for v in ready["variants"]} + check(set(meta) == {"thumbnail", "crop", "optimized"}, f"{kind}: variants {sorted(meta)}") + for name, size in sizes.items(): + check([meta[name]["width"], meta[name]["height"]] == size, + f"{kind}/{name}: {meta[name]['width']}x{meta[name]['height']}, expected {size}") + check(meta["thumbnail"]["content_type"] == "image/jpeg" and meta["crop"]["content_type"] == "image/jpeg" + and meta["optimized"]["content_type"] == "image/webp", f"{kind}: wrong variant content types") + original = content(att["id"]) + check(original.data == data, f"{kind}: original is not byte-identical") + for name in ("thumbnail", "crop", "optimized"): + reply = content(att["id"], name) + check(reply.headers["content-type"] == meta[name]["content_type"], f"{kind}/{name}: Content-Type") + info = assert_one_result(att["id"], kind) + for name, v in info["variants"].items(): + check(v["exif_tags"] == 0 and not v["has_xmp"], f"{kind}/{name}: metadata not stripped") + if kind == "exif6": + top = info["variants"]["thumbnail"]["pixel_top_center"] + check(top[0] > 200 and top[1] < 60, f"exif6: orientation not applied, top pixel {top}") + if kind == "alpha": + left = info["variants"]["thumbnail"]["pixel_left_center"] + right = info["variants"]["thumbnail"]["pixel_right_center"] + check(left[:3] == [255, 255, 255] and right[2] > 200, f"alpha: not composited on white {left} {right}") + check(info["variants"]["optimized"]["mode"] == "RGBA", "alpha: optimized WebP lost transparency") + log(f"{kind}: " + ", ".join(f"{n} {s[0]}x{s[1]}" for n, s in sizes.items()) + + f"; original byte-identical; EXIF/XMP in variants: 0; S3 keys {info['final_keys']}") + # Отсутствующий вариант у ready -> 404 (нет такого имени в контракте -> 400), queued -> 409 проверяется ниже. + content(att["id"], "unknown", expected=400) + log("unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload") + + +def test_limits(): + """Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.""" + alice, _bob, chat = user_and_chat() + big = b"\x89PNG\r\n\x1a\n" + b"\0" * (10 * 1024 * 1024) + call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=big.png", 413, raw=big, token=alice["token"], + headers={"Content-Type": "image/png"}, timeout=60) + log("10 MiB + 8 bytes -> 413 before any job is created") + report = probe("imaging-limits") + at, over = report["at_limit_40mpx"], report["over_limit_max_10mpx"] + check(at["exit_code"] == 0, f"40 Mpx PNG at the limit must be processed, exit {at['exit_code']}") + check(over["exit_code"] == 4, f"worker-side pixel limit must reject (exit 4), got {over['exit_code']}") + check(at["peak_rss_mib_max_children"] < report["memory_limit_mib"], + f"peak RSS {at['peak_rss_mib_max_children']} MiB exceeds RLIMIT_AS {report['memory_limit_mib']} MiB") + log(f"imaging.py on 8000x5000 PNG: exit 0, peak RSS {at['peak_rss_mib_max_children']} MiB " + f"(RLIMIT_AS {report['memory_limit_mib']} MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)") + # Память: процесс обработки упирается в RLIMIT_AS -> MemoryError -> постоянная ошибка resource_limit. + fault("imaging_oom") + att = upload(image("thumb320"), filename="oom.png") + failed = wait_status(att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "resource_limit", + f"oom: expected failed/resource_limit, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == 1, "resource_limit is permanent: no retries expected") + log(f"memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived") + STATE["oom_job"] = j["job_id"] + # Время: зависший процесс обработки убивается по JOB_TIMEOUT_SECONDS -> временная ошибка -> retry -> ready. + fault("imaging_hang") + att = upload(image("thumb320"), filename="hang.png") + started = time.monotonic() + ready = wait_status(att["id"], "ready", TIMEOUT + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check("lease_expired" not in names(evs), f"a long-running (alive) attempt must keep its lease: {evs}") + check(any("processing_timeout" in e for e in evs) and j["attempt"] == 2, f"timeout not handled as retry: {evs}") + log(f"imaging hang -> killed after {TIMEOUT} s (processing_timeout, temporary), lease renewed meanwhile " + f"(no lease_expired), attempt 2 ready after {time.monotonic() - started:.0f} s") + + +def test_worker_down(): + """Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.""" + alice, bob, chat = user_and_chat() + compose("stop", "worker") + log("worker stopped") + data = image("thumb320") + att = upload(data, filename="offline.png") + log(f"upload while worker is down -> 202 queued, attachment {att['id']}") + j = job(att["id"]) + source_key = sql_value(f"SELECT source_key FROM attachments WHERE id = '{att['id']}'") + check(source_key.startswith("staging/"), "source must be in the durable staging prefix") + check(probe("exists", source_key)["exists"], "staging object must exist before 202") + msg = post_message(alice, chat["id"], "photo while worker is down", [att["id"]]).json + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + content(att["id"], expected=409) + content(att["id"], "thumbnail", expected=409) + send(bob, chat["id"], "other messages keep working") + deadline = time.monotonic() + 10 + while outbox_rows(j["job_id"]) != ["1:true"]: + check(time.monotonic() < deadline, "dispatcher did not publish the job") + time.sleep(0.2) + deadline = time.monotonic() + 15 # счётчики quorum-очереди в rabbitmqctl обновляются раз в несколько секунд + while (queue_depth().get("attachments.process.v1") or (0, 0))[0] < 1: + check(time.monotonic() < deadline, "published message is not visible in the queue") + time.sleep(0.5) + depth = queue_depth().get("attachments.process.v1") + log(f"message with attachment 201, content 409, other messages 201; job {j['job_id']} published, " + f"queue messages/unacked = {depth}") + compose("restart", "rabbitmq") + compose("restart", "api") + wait_ready() + depth_after = queue_depth().get("attachments.process.v1") + check(depth_after and depth_after[0] >= 1, f"message lost after broker restart: {depth_after}") + check(get(att["id"])["status"] == "queued", "job must still be queued after restarts") + log(f"rabbitmq and api restarted with volumes; queue still holds {depth_after[0]} message(s); status queued") + compose("start", "worker") + ready = wait_status(att["id"], "ready", 60) + j2 = job(att["id"]) + check(j2["job_id"] == j["job_id"] and j2["status"] == "ready", "same job_id must finish") + thumb = content(att["id"], "thumbnail") + check(len(thumb.data) == {v["name"]: v for v in ready["variants"]}["thumbnail"]["size_bytes"], "thumbnail size") + info = assert_one_result(att["id"], "worker-down") + check(not info["staging_present"] and not probe("exists", source_key)["exists"], + "staging must be cleaned after the safe commit") + log(f"worker started -> same job {j['job_id']} ready, thumbnail downloadable; staging object deleted") + + +def test_broker_down(): + """Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.""" + compose("stop", "rabbitmq") + log("rabbitmq stopped") + att = upload(image("thumb320"), filename="no-broker.png") + call("GET", "/health/ready", 200) + j = job(att["id"]) + time.sleep(3) + check(outbox_rows(j["job_id"]) == ["1:false"], "outbox row must wait while the broker is down") + log(f"upload -> 202 and /health/ready 200 without broker; outbox row pending for job {j['job_id']}") + compose("start", "rabbitmq") + wait_status(att["id"], "ready", 90) + check(outbox_rows(j["job_id"]) == ["1:true"], "outbox row must be published after recovery") + assert_one_result(att["id"], "broker-down") + log("rabbitmq started -> dispatcher published the pending row -> ready") + + +def test_outbox_crash(): + """Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.""" + fault("dispatcher_crash_before_publish") + att = upload(image("thumb320"), filename="crash-before-publish.png") + ready = wait_status(att["id"], "ready", 60) + j = job(att["id"]) + logs = compose_output("logs", "--tail", "400", "dispatcher") + check("FAULT INJECTION: dispatcher_crash_before_publish" in logs, "fault did not fire") + check(outbox_rows(j["job_id"]) == ["1:true"] and j["attempt"] == 1, "outbox must publish exactly the same job") + assert_one_result(att["id"], "crash-before-publish") + log(f"dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job " + f"{j['job_id']} ready, attempt 1, no manual action") + fault("dispatcher_crash_after_confirm") + att = upload(image("thumb320"), filename="crash-after-confirm.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + deadline = time.monotonic() + 20 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"duplicate delivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + assert_one_result(att["id"], "crash-after-confirm") + log(f"dispatcher killed after publisher confirm, before marking outbox -> message published twice; " + f"worker: {names(events(j['job_id']))} -> one result set") + + +def test_crash_after_put(): + """Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.""" + fault("worker_crash_after_put") + att = upload(image("thumb320"), filename="crash-after-put.png") + wait_status(att["id"], "ready", LEASE + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check(names(evs).count("claimed") == 2 and "lease_expired" in names(evs), f"unexpected history {evs}") + assert_one_result(att["id"], "crash-after-put") + log(f"worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired " + f"after {LEASE} s, attempt {j['attempt']} rewrote the same keys -> one set; events {names(evs)}") + + +def test_crash_after_commit(): + """Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.""" + fault("worker_crash_after_commit") + att = upload(image("thumb320"), filename="crash-after-commit.png") + wait_status(att["id"], "ready", 30) + j = job(att["id"]) + deadline = time.monotonic() + 30 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"redelivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + evs = events(j["job_id"]) + check(names(evs).count("ready") == 1 and j["attempt"] == 1, f"ready must be recorded once: {evs}") + assert_one_result(att["id"], "crash-after-commit") + deadline = time.monotonic() + 20 # счётчики quorum-очереди обновляются с задержкой + while queue_depth().get("attachments.process.v1") != (0, 0): + check(time.monotonic() < deadline, f"message left in the queue: {queue_depth()}") + time.sleep(0.5) + log(f"worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op ({evs[-1]}); " + f"queue empty (0 ready, 0 unacked)") + STATE["staging_check"] = att["id"] + + +def test_lease(): + """Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; + результат зависшего отклоняется fencing-токеном, ready не откатывается.""" + fault("worker_freeze") + att = upload(image("thumb320"), filename="frozen.png") + started = time.monotonic() + wait_status(att["id"], "ready", LEASE + 40) + took = time.monotonic() - started + j = job(att["id"]) + deadline = time.monotonic() + 3 * LEASE + 20 + while "stale_result_rejected" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"stale result of the frozen attempt not seen: {events(j['job_id'])}") + time.sleep(0.5) + evs = events(j["job_id"]) + check(get(att["id"])["status"] == "ready", "late result must not roll ready back") + check(names(evs).count("ready") == 1, f"exactly one ready: {evs}") + assert_one_result(att["id"], "lease") + log(f"frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after " + f"{took:.0f} s; frozen attempt woke up -> stale_result_rejected; events {names(evs)}") + + +def test_retry(): + """Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.""" + fault("worker_s3_error", 2) + att = upload(image("thumb320"), filename="retry.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + evs = [e for e in events(j["job_id"]) if ":retry_scheduled:" in e] + check(len(evs) == 2 and j["attempt"] == 3, f"expected 2 retries, got {evs}") + delays = [float(e.split("delay=")[1].rstrip("s")) for e in evs] + check(1.0 <= delays[0] <= 2.0 and 2.0 <= delays[1] <= 4.0, f"backoff out of range: {delays}") + assert_one_result(att["id"], "retry") + log(f"2 x storage_unavailable -> retries after {delays[0]:.2f} s and {delays[1]:.2f} s (equal jitter: " + f"1..2 s, 2..4 s), attempt 3 ready") + + +def test_exhausted_replay(): + """Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.""" + alice, _bob, chat = user_and_chat() + fault("worker_s3_error", MAX_ATTEMPTS) + att = upload(image("thumb320"), filename="exhausted.png") + failed = wait_status(att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", + f"expected failed/retry_exhausted, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == MAX_ATTEMPTS, f"attempts {j['attempt']}") + content(att["id"], expected=409) + post_message(alice, chat["id"], "", [att["id"]], expected=409) + registry = {row["job_id"]: row for row in ops("failed")} + check(registry.get(j["job_id"], {}).get("replayable") is True, "failed job must be in the registry") + log(f"{MAX_ATTEMPTS} temporary errors -> failed/retry_exhausted (public error_code), content 409, " + f"attach 409, listed by ops.py failed (replayable)") + replay = ops("replay", j["job_id"], "--reason", "s3 recovered") + check("replayed" in replay, f"replay refused: {replay}") + wait_status(att["id"], "ready", 60) + assert_one_result(att["id"], "replay") + j2 = job(att["id"]) + refused = ops("replay", j["job_id"]) + check("refused" in refused, "replay of a ready job must be refused") + again = job(att["id"]) + check(again == j2, "refused replay must not change the job") + check(names(events(j["job_id"])).count("replayed") == 1, "audit must contain the replay once") + log(f"ops replay -> same job_id {j['job_id']} attempt {j2['attempt']} ready; second replay refused: " + f"\"{refused['refused']}\"") + + +def test_permanent(): + """Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.""" + fault("imaging_invalid") + att = upload(image("thumb320"), filename="invalid.png") + failed = wait_status(att["id"], ("failed", "ready"), 30) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", f"got {failed}") + j = job(att["id"]) + check(j["attempt"] == 1 and "retry_scheduled" not in names(events(j["job_id"])), "no retries for permanent") + log("invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)") + + +def test_concurrency(): + """Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.""" + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_consumers", "-p", "messenger", + "queue_name", "prefetch_count", "ack_required") + lines = [line.split() for line in out.splitlines() if line.strip()] + header = lines[0] if lines and lines[0][0] == "queue_name" else ["queue_name", "prefetch_count", "ack_required"] + consumers = [dict(zip(header, line)) for line in lines if line[0] == "attachments.process.v1"] + check(len(consumers) == 2 and all(c["prefetch_count"] == "1" and c["ack_required"] == "true" + for c in consumers), f"consumers {out}") + # Копим очередь при остановленном worker, затем запускаем: работы больше, чем потоков. + compose("stop", "worker") + data = image("large") + ids = [upload(data, "image/jpeg", f"burst{n}.jpg")["id"] for n in range(8)] + id_list = ", ".join(f"'{i}'" for i in ids) + compose("start", "worker") + started = time.monotonic() + peak, samples = 0, 0 + while True: + busy, ready = map(int, sql_value( + f"SELECT count(*) FILTER (WHERE status = 'processing') || '|' || count(*) FILTER (WHERE status = 'ready') " + f"FROM attachment_jobs WHERE attachment_id IN ({id_list})").split("|")) + peak, samples = max(peak, busy), samples + 1 + if ready == len(ids): + break + check(time.monotonic() - started < 120, "burst not processed in 120 s") + time.sleep(0.05) + took = time.monotonic() - started + check(peak == 2, f"peak {peak} jobs in processing, expected exactly WORKER_CONCURRENCY=2") + log(f"2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in {took:.1f} s after " + f"worker start, peak processing {peak} of 8 ({samples} samples)") + + +def test_dlq(): + """Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.""" + publish = r''' +import json, sys, uuid +import broker, pipeline +p = broker.Publisher() +p.open().basic_publish(pipeline.EXCHANGE, pipeline.ROUTING_KEY, b'{"not":"a job"}', mandatory=True) +p.publish(pipeline.build_message(uuid.uuid4(), uuid.uuid4(), 1, 1, None)) +p.close() +print("{}") +''' + before = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + in_container("dispatcher", publish) + deadline = time.monotonic() + 15 + while queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] < before + 2: + check(time.monotonic() < deadline, "messages did not reach the DLQ") + time.sleep(0.3) + peek = ops("dlq", "--limit", "50") + reasons = {(m["invalid"] is not None, m["dead_letter_reason"]) for m in peek} + check((True, "rejected") in reasons, f"invalid message not in DLQ: {peek}") + replayed = ops("dlq-replay", "--limit", "50") + kept = [m for m in replayed if m["action"] == "keep"] + check(len(kept) >= 2, "invalid/unknown messages must be kept, not executed") + after = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + check(after >= before + 2, "dlq-replay must not drop messages it cannot handle") + log(f"schema-invalid message and unknown job_id -> reject -> DLQ ({after} message(s)); " + f"dlq-replay kept them: {[m['note'] for m in kept][:2]}") + + +def test_staging_cleanup(): + """Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).""" + att_id = STATE.get("staging_check") + if not att_id: + log("skip: run crash-after-commit first") + return + key = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + if key: + log(f"staging source still present after the crash (worker died before cleanup); waiting 60 s window") + time.sleep(62) + from common import janitor + report = janitor() + # фоновый janitor (проход раз в JANITOR_INTERVAL_SECONDS) мог успеть первым в окне 62 с — проверяем результат + left = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + check(not left, f"staging source still referenced: {report}") + check(not probe("exists", key)["exists"], "staging object must be gone") + by = "this pass" if report["staging_ready_deleted"] else "the background janitor pass" + log(f"janitor removed the staging source of the ready attachment ({by}); source_key NULL, object gone") + else: + log("staging already cleaned by the worker") + + +SCENARIOS = { + "variants": test_variants, "limits": test_limits, "worker-down": test_worker_down, + "broker-down": test_broker_down, "outbox-crash": test_outbox_crash, "crash-after-put": test_crash_after_put, + "crash-after-commit": test_crash_after_commit, "staging": test_staging_cleanup, "lease": test_lease, + "retry": test_retry, "exhausted-replay": test_exhausted_replay, "permanent": test_permanent, + "concurrency": test_concurrency, "dlq": test_dlq, +} + + +def main(): + chosen = sys.argv[1:] or list(SCENARIOS) + wait_ready() + clear_faults() + tests = [] + for name in chosen: + def run(name=name): + log(f"--- {name}: {SCENARIOS[name].__doc__.splitlines()[0]}") + try: + SCENARIOS[name]() + finally: + compose("start", "worker") # идемпотентно: стенд остаётся рабочим даже после FAIL + compose("start", "rabbitmq") + run.__name__ = name + tests.append(run) + run_tests(tests) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 0000000..4af44fa --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) diff --git a/student-tests/test_s3_security.py b/student-tests/test_s3_security.py new file mode 100644 index 0000000..ece71a3 --- /dev/null +++ b/student-tests/test_s3_security.py @@ -0,0 +1,146 @@ +"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен. + +Нужен запущенный стенд из compose.yaml. +""" +import json +import socket + +import images +from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload + +# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API. +ANONYMOUS = r''' +import json, sys, urllib.request, urllib.error +import settings +base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET +def status(method, url, data=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r: + return r.status, r.read(300).decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read(300).decode("utf-8", "replace") +out = {} +for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None), + ("head_object", "HEAD", base + "/" + sys.argv[1], None), + ("list_bucket", "GET", base + "?list-type=2", None), + ("put_object", "PUT", base + "/originals/anonymous-test", b"x"), + ("get_bucket_policy", "GET", base + "?policy", None)): + code, body = status(method, url, data) + out[name] = {"status": code, "s3_code": body.split("")[1].split("")[0] if "" in body else ""} +print(json.dumps(out)) +''' + +# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json). +API_KEY_RIGHTS = r''' +import io, json, sys, uuid +from botocore.exceptions import ClientError +import settings, storage +c, bucket = storage._client, settings.S3_BUCKET +def attempt(name, fn): + try: + fn(); return "allowed" + except ClientError as e: + return "denied: " + e.response.get("Error", {}).get("Code", "?") +public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) +key = "originals/" + uuid.uuid4().hex +result = { + "get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()), + "put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")), + "delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)), + "list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)), + "create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])), + "make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)), + "read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)), + "delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)), + "change versioning": attempt("versioning", lambda: c.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})), +} +try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket + result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", [])) +except ClientError as e: + result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?") +print(json.dumps(result)) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + att = upload(alice, chat["id"], images.sample_png()).json + key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + return alice, bob, att, key + + +def test_bucket_is_private(): + alice, bob, att, key = prepare() + result = last_json(app_python(ANONYMOUS, key)) + for name, reply in result.items(): + print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}") + check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}") + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + print("the same file through the API for a chat member -> 200") + print("PASS: the bucket is private: no anonymous read, list, write or policy read") + + +def test_api_key_is_limited(): + alice, bob, att, key = prepare() + result = last_json(app_python(API_KEY_RIGHTS, key)) + allowed = {"get own object", "put object", "delete object", "list bucket"} + visible = result.pop("visible buckets") + print(f"API key: {'list buckets':<32} -> {visible}") + check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets") + for name, outcome in result.items(): + print(f"API key: {name:<32} -> {outcome}") + if name in allowed: + check(outcome == "allowed", f"API key must be allowed to {name}") + else: + check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}") + print("PASS: the API key can only get/put/delete/list objects of its own bucket; " + "bucket admin, policy and other buckets are denied") + + +def test_no_open_ports_or_console(): + config = json.loads(compose_output("config", "--format", "json")) + s3 = config["services"]["s3"] + check(not s3.get("ports"), "s3 must not publish ports") + check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, " + "filer UI 8888 are reachable only inside the stand)") + for port in (8333, 9333, 8888, 19333): + sock = socket.socket() + sock.settimeout(1) + refused = sock.connect_ex(("127.0.0.1", port)) != 0 + sock.close() + check(refused, f"127.0.0.1:{port} is reachable from the host") + print(f"127.0.0.1:{port} from the host -> connection refused") + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()} + for name, items in sorted(secrets.items()): + print(f"secrets of {name}: {items or '-'}") + for name in ("api", "janitor"): + check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"}, + f"{name} must not get S3 admin keys or the key config") + check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key") + print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init") + + +def test_bootstrap_is_idempotent(): + alice, bob, att, key = prepare() + output = compose_output("run", "--rm", "--no-deps", "s3-init") + print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip())) + check("bucket is private" in output, "bootstrap did not confirm a private bucket") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "file changed after repeated bootstrap") + upload(alice, att["chat_id"], images.sample_png()) + print("after the repeated bootstrap: old file downloads, new upload works") + print("PASS: bucket/user/policy bootstrap can be repeated safely") + + +if __name__ == "__main__": + run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console, + test_bootstrap_is_idempotent]) diff --git a/student-tests/test_telemetry.py b/student-tests/test_telemetry.py new file mode 100644 index 0000000..f6442fd --- /dev/null +++ b/student-tests/test_telemetry.py @@ -0,0 +1,258 @@ +"""Lab6: связь запроса, структурного лога и трассы; сквозная трасса upload -> worker, включая retry. + + python student-tests/test_telemetry.py # все сценарии + python student-tests/test_telemetry.py upload-chain # один (список SCENARIOS ниже) + +Нужен запущенный стенд (compose.yaml, FAULT_INJECTION=1 для retry/replay). Клиент сам задаёт W3C +traceparent запроса, поэтому трасса находится в Tempo по известному trace_id, без поиска «похожей». +Тест печатает дерево спанов (без данных пользователя) — это доказательство для отчёта. + +Сценарии: + request уровень 3: POST сообщения -> server-спан API с дочерними спанами PostgreSQL/Redis, + запись лога event=http.request с тем же trace_id/span_id и request_id из X-Request-Id; + текст сообщения, пароль и токен не попали ни в логи (Loki), ни в атрибуты спанов. + upload-chain уровень 4: одна трасса API(server) -> create(producer, outbox) -> dispatcher publish + (producer) -> worker process (consumer) -> S3 PutObject x4 и COMMIT ready; логи worker + с тем же trace_id. + retry-chain уровень 4: временная ошибка S3 в worker (fault worker_s3_error) -> в той же трассе + попытка 1 (ERROR, outcome=retry) -> новая публикация -> попытка 2 (ready). + replay-link уровень 4: попытки исчерпаны -> failed -> ops.py replay создаёт НОВУЮ трассу со + ссылкой (link) на исходную загрузку, worker завершает её в ready. + history уровень 3: после restart prometheus/loki/tempo (volumes сохранены) прежние трасса, + логи и метрика по-прежнему находятся. +""" +import json +import subprocess +import sys +import time +import uuid + +import obs +from common import ROOT, call, check, compose, new_chat, new_user, run_tests, wait_attachment +from images import png + +LOKI_WAIT = 60 +TRACE_WAIT = 60 + + +def upload_traced(user, chat_id, data, traceparent, filename="trace.png"): + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", 202, raw=data, + token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent}) + return reply + + +def by_name(spans, name, service=None): + return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)] + + +def parent_of(spans, span): + return next((s for s in spans if s["span_id"] == span["parent_id"]), None) + + +def fault(point, count=1): + for n in range(count): + subprocess.run(["docker", "compose", "exec", "-T", "worker", "sh", "-c", f"touch /faults/{point}.{n + 1}"], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def ops(*args): + result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], + cwd=ROOT, capture_output=True, check=True) + return json.loads(result.stdout.decode("utf-8")) + + +# ---------- сценарии ---------- + +def test_request(): + user = new_user("trace") + chat = new_chat(user, title="Trace chat") + marker = f"secret-text-{uuid.uuid4().hex}" + trace_id, traceparent = obs.new_traceparent() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": marker}, + headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}) + request_id = reply.headers["x-request-id"] + check(reply.headers.get("x-trace-id") == trace_id, "X-Trace-Id must continue the client's traceparent") + print(f"POST message: 201, X-Request-Id={request_id}, trace_id={trace_id} (client traceparent)") + + spans = obs.wait_trace(trace_id, lambda s: by_name(s, "POST /api/v1/chats/{chat_id}/messages", "messenger-api") + and any(x["kind"] == "CLIENT" and x["name"] == "INSERT" for x in s), TRACE_WAIT) + server = by_name(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")[0] + check(server["kind"] == "SERVER" and server["attrs"].get("http.response.status_code") == 201, "server span") + check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span") + db = [s for s in spans if s["parent_id"] == server["span_id"] and "db.statement" in s["attrs"] + or "db.query.text" in s["attrs"]] + check(db, "PostgreSQL client spans under the server span") + check(by_name(spans, "redis GET", "messenger-api"), "Redis session lookup span") + print(obs.span_tree(spans)) + + rows = obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}" | event="http.request"', + 1, LOKI_WAIT) + labels, line = rows[0] + check(labels.get("request_id") == request_id, "log request_id == X-Request-Id") + check(labels.get("span_id") == server["span_id"], "log span_id == server span id") + for field in ("service_name", "severity_text", "event", "trace_id", "span_id"): + check(labels.get(field), f"log field {field} is missing") + print(f"Loki: {line!r} event={labels['event']} severity={labels['severity_text']} " + f"service={labels['service_name']} trace_id={labels['trace_id']} span_id={labels['span_id']} " + f"request_id={labels['request_id']}") + + # Ни текст сообщения, ни пароль, ни токен не попали в телеметрию. + for secret, what in ((marker, "message text"), (user["password"], "password"), (user["token"][:24], "token")): + found = obs.loki_query(f'{{service_name=~"messenger-.*"}} |= `{secret}`', minutes=10, limit=5) + check(not found, f"{what} found in Loki logs") + check(not any(secret in str(v) for s in spans for v in s["attrs"].values()), f"{what} found in span attributes") + check(not any(k.lower().startswith(("http.request.header.authorization", "enduser")) for s in spans + for k in s["attrs"]), "Authorization/enduser attributes in spans") + print("PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id " + "and request_id; no message text, password or token in logs/span attributes") + + +def test_upload_chain(): + user = new_user("chain") + chat = new_chat(user, title="Chain chat") + trace_id, traceparent = obs.new_traceparent() + accepted = upload_traced(user, chat["id"], png(320, 200), traceparent).json + ready = wait_attachment(user, accepted["id"]) + check(ready["status"] == "ready", f"attachment is {ready['status']}") + + def complete(spans): + return by_name(spans, "commit result", "messenger-worker") and by_name(spans, "cleanup staging") + + spans = obs.wait_trace(trace_id, complete, TRACE_WAIT) + print(obs.span_tree(spans)) + server = by_name(spans, "POST /api/v1/chats/{chat_id}/attachments", "messenger-api")[0] + create = by_name(spans, "create attachments", "messenger-api")[0] + publish = by_name(spans, "publish attachments", "messenger-dispatcher")[0] + consume = by_name(spans, "process attachments.process.v1", "messenger-worker")[0] + check(create["kind"] == "PRODUCER" and publish["kind"] == "PRODUCER" and consume["kind"] == "CONSUMER", "kinds") + check(parent_of(spans, create)["span_id"] == server["span_id"], "create is a child of the API request") + check(publish["parent_id"] == create["span_id"], "publish (dispatcher) is a child of create (API outbox)") + check(consume["parent_id"] == publish["span_id"], "consume (worker) is a child of publish (W3C via AMQP headers)") + check(consume["attrs"].get("messenger.job.outcome") == "ready", "consumer outcome ready") + puts = [s for s in by_name(spans, "S3 PutObject", "messenger-worker") + if s["attrs"].get("aws.s3.key", "").startswith(f"attachments/{accepted['id']}/v1/")] + check(len(puts) == 4, f"4 PUTs of the final objects, got {len(puts)}") + check({s["service"] for s in spans} >= {"messenger-api", "messenger-dispatcher", "messenger-worker"}, "services") + events = {labels["event"] for labels, _ in obs.wait_logs( + f'{{service_name="messenger-worker"}} | trace_id="{trace_id}"', 2, LOKI_WAIT)} + check({"job.claimed", "job.ready"} <= events, f"worker logs with this trace_id: {events}") + print(f"worker logs with trace_id {trace_id}: {sorted(events)}") + print(f"PASS: one trace {trace_id}: API 202 -> outbox create -> dispatcher publish -> worker consume -> " + f"4 S3 objects + COMMIT ready ({len(spans)} spans, services: " + f"{', '.join(sorted({s['service'] for s in spans}))})") + + +def test_retry_chain(): + user = new_user("retry") + chat = new_chat(user, title="Retry chat") + before = obs.prom_value('sum(messenger_job_duration_seconds_count{outcome="retry"})', 0.0) + fault("worker_s3_error", 1) + trace_id, traceparent = obs.new_traceparent() + accepted = upload_traced(user, chat["id"], png(64, 48), traceparent).json + ready = wait_attachment(user, accepted["id"]) + check(ready["status"] == "ready", f"attachment is {ready['status']}") + + def complete(spans): + return len(by_name(spans, "process attachments.process.v1")) >= 2 and by_name(spans, "commit result") + + spans = obs.wait_trace(trace_id, complete, TRACE_WAIT) + print(obs.span_tree(spans)) + attempts = sorted(by_name(spans, "process attachments.process.v1", "messenger-worker"), key=lambda s: s["start"]) + first, second = attempts[0], attempts[-1] + check(first["status"] == "ERROR" and first["attrs"].get("messenger.job.outcome") == "retry", "attempt 1 = retry") + check(first["attrs"].get("error.type") == "storage_unavailable", "attempt 1 error.type") + check(second["attrs"].get("messenger.job.outcome") == "ready", "attempt 2 = ready") + retry_create = [s for s in by_name(spans, "create attachments") if s["parent_id"] == first["span_id"]] + check(retry_create, "retry message is created inside attempt 1 (outbox in the same transaction)") + second_publish = parent_of(spans, second) + check(second_publish["name"] == "publish attachments" and second_publish["parent_id"] == retry_create[0]["span_id"], + "attempt 2 consumes the retry publication") + check(len(by_name(spans, "publish attachments")) == 2, "two publications in one trace") + rows = obs.wait_logs(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="job.retry"', 1, LOKI_WAIT) + print(f"worker log: {rows[0][1]!r} error_type={rows[0][0].get('error_type')}") + after = obs.wait_metric('sum(messenger_job_duration_seconds_count{outcome="retry"})', lambda v: v > before, 40) + print(f"messenger_job_duration_seconds_count{{outcome=\"retry\"}}: {before:g} -> {after:g}") + print(f"PASS: retry in one trace {trace_id}: attempt 1 ERROR storage_unavailable -> create -> publish -> " + f"attempt 2 ready; retry counted in metrics and logged") + + +def test_replay_link(): + user = new_user("replay") + chat = new_chat(user, title="Replay chat") + fault("worker_s3_error", 5) + trace_id, traceparent = obs.new_traceparent() + accepted = upload_traced(user, chat["id"], png(32, 32), traceparent).json + failed = wait_attachment(user, accepted["id"], statuses=("failed",), seconds=120) + check(failed["error_code"] == "retry_exhausted", f"error_code {failed['error_code']}") + job = next(j for j in ops("failed", "--limit", "20") if j["attachment_id"] == accepted["id"]) + result = ops("replay", job["job_id"], "--reason", "lab6 link test") + check("replayed" in result, f"replay refused: {result}") + ready = wait_attachment(user, accepted["id"]) + check(ready["status"] == "ready", "ready after replay") + original = obs.wait_trace(trace_id, lambda s: len(by_name(s, "process attachments.process.v1")) >= 5, TRACE_WAIT) + print(f"original trace {trace_id}: {len(by_name(original, 'process attachments.process.v1'))} attempts, " + f"last outcome {sorted(by_name(original, 'process attachments.process.v1'), key=lambda s: s['start'])[-1]['attrs'].get('messenger.job.outcome')}") + found = None + deadline = time.monotonic() + TRACE_WAIT + while found is None and time.monotonic() < deadline: + for item in obs.tempo_search(f'{{ name = "create attachments" && span.messenger.job.id = "{job["job_id"]}" }}', + minutes=15, limit=20): + if item["traceID"].rjust(32, "0") != trace_id: + found = item["traceID"].rjust(32, "0") + if found is None: + time.sleep(2) + check(found, "replay trace not found") + spans = obs.wait_trace(found, lambda s: by_name(s, "commit result"), TRACE_WAIT) + print(obs.span_tree(spans)) + create = by_name(spans, "create attachments")[0] + linked = {link["trace_id"] for link in create["links"]} + check(trace_id in linked, f"create span of the replay links to the original trace {trace_id}: {linked}") + print(f"replay trace {found}: create span links -> {sorted(linked)} (original upload trace {trace_id})") + check(by_name(spans, "ops replay"), "ops replay span is the root of the new trace") + check(by_name(spans, "process attachments.process.v1")[0]["attrs"].get("messenger.job.outcome") == "ready", + "replayed attempt ready") + print("PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it") + + +def test_history(): + user = new_user("history") + chat = new_chat(user, title="History chat") + trace_id, traceparent = obs.new_traceparent() + call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": "history"}, + headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}) + obs.wait_trace(trace_id, seconds=TRACE_WAIT) + obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"', 1, LOKI_WAIT) + query = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})' + stamp = round(time.time() - 30) # значение на момент t-30 с (параметр time запроса API Prometheus) + rows = obs.prom_query(query, at=stamp) + before = rows[0][1] if rows else None + check(before, "metric sample before restart") + print(f"before restart: trace {trace_id} found, its log found, {query} at t-30s = {before:g}") + compose("restart", "prometheus", "loki", "tempo") + deadline = time.monotonic() + 120 + while True: + try: + spans = obs.tempo_trace(trace_id) + logs = obs.loki_query(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"') + rows = obs.prom_query(query, at=stamp) + after = rows[0][1] if rows else None + if spans and logs and after == before: + break + except OSError: + pass + check(time.monotonic() < deadline, "history not available 120 s after restart") + time.sleep(3) + print(f"after restart of prometheus, loki, tempo: trace {trace_id} ({len(spans)} spans), " + f"{len(logs)} log record(s), metric at t-30s = {after:g} (same value)") + print("PASS: persistent volumes keep traces, logs and metrics across restarts of the stores") + + +SCENARIOS = {"request": test_request, "upload-chain": test_upload_chain, "retry-chain": test_retry_chain, + "replay-link": test_replay_link, "history": test_history} + +if __name__ == "__main__": + names = sys.argv[1:] or list(SCENARIOS) + unknown = [n for n in names if n not in SCENARIOS] + if unknown: + raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}") + run_tests([SCENARIOS[n] for n in names]) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..01f1ced --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,222 @@ +"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик. + +Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что +тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково +(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше +DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование) +при тех же 10 загрузках не помещается в лимит 512 MiB. + +Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков +одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status +(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера +(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с. + + python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json] +""" +import argparse +import hashlib +import json +import platform +import statistics +import threading +import time + +import images +from common import (ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_attachment, + wait_balanced, wait_ready) + + +def proc_status(container): + """VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты).""" + script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; " + "for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current " + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; " + "do [ -r $f ] && echo \"$f $(cat $f)\"; done; true") + out = docker("exec", container, "sh", "-c", script) + values = {} + for line in out.splitlines(): + parts = line.replace(":", " ").split() + if parts[0] in ("VmRSS", "VmHWM"): + values[parts[0]] = int(parts[1]) + elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")): + values["cgroup_peak"] = int(parts[1]) + elif parts[0].endswith(("memory.current", "usage_in_bytes")): + values["cgroup_current"] = int(parts[1]) + return values + + +class Sampler(threading.Thread): + def __init__(self, containers): + super().__init__(daemon=True) + self.containers = containers + self.stop = threading.Event() + self.rss = {c: 0 for c in containers} + self.cgroup = {c: 0 for c in containers} + self.samples = 0 + + def run(self): + while not self.stop.is_set(): + for c in self.containers: + try: + v = proc_status(c) + except Exception: # noqa: BLE001 — один пропущенный замер не важен + continue + self.rss[c] = max(self.rss[c], v.get("VmRSS", 0)) + self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0)) + self.samples += 1 + + +def api_containers(): + from common import api_containers as running + return running() + + +def fresh_replicas(): + compose("restart", "api") + containers = wait_balanced(2, seconds=90) + wait_ready() + return containers + + +def warm_up(user, chat): + """По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п.""" + for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4: + upload(user, chat["id"], data, ctype) + + +def scenario(name, payloads, concurrency): + user = new_user("mem") + chat = new_chat(user) + containers = fresh_replicas() + warm_up(user, chat) + before = {c: proc_status(c) for c in containers} + sampler = Sampler(containers) + sampler.start() + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(i): + data, ctype = payloads[i % len(payloads)] + barrier.wait() + started = time.monotonic() + try: + # lab5: замеряем только приём (202 queued); обработку worker'ом ждём ниже, вне замера + reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180, wait=False) + results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data) + except AssertionError as exc: + results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] + wall = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - wall + time.sleep(0.5) + sampler.stop.set() + sampler.join() + after = {c: proc_status(c) for c in containers} + + ok = [r for r in results if r[0] == 202] + check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 202]}") + for status, _, _, att, data in ok[:3]: + att = wait_attachment(user, att["id"], seconds=120) + check(att["status"] == "ready", f"{name}: attachment {att['id']} is {att['status']}, expected ready") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"]) + check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs") + latencies = sorted(r[1] for r in results) + per_replica = [] + for c in containers: + b, a = before[c], after[c] + per_replica.append({ + "replica": c[:12], + "uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])), + "rss_before_mib": round(b["VmRSS"] / 1024, 1), + "rss_peak_mib": round(a["VmHWM"] / 1024, 1), + "rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1), + "rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1), + "cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None, + "cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None, + }) + sizes = sorted({len(d) for d, _ in payloads}) + summary = { + "scenario": name, + "concurrency": concurrency, + "file_bytes": sizes, + "total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1), + "all_202": True, + "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)}, + "sampler_rounds": sampler.samples, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False)) + return summary + + +# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование. +NAIVE_VS_STREAMING = r''' +import json, os, subprocess, sys +code = r""" +import io, os, resource, struct, sys, zlib +sys.path.insert(0, os.getcwd()) +os.environ.setdefault("S3_ACCESS_KEY_ID", "x") +from PIL import Image +import uploads # импорт до замера: сравниваем только саму проверку файла +Image.MAX_IMAGE_PIXELS = None +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +w, h = 8000, 5000 +comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3) +idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush() +data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") +del idat, comp, row +base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if sys.argv[1] == "naive": + body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB) + im = Image.open(io.BytesIO(data)); im.load() +else: + f = io.BytesIO(data); uploads.check_png(f) +print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024) +""" +out = {} +for mode in ("naive", "streaming"): + r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd()) + out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:] +print(json.dumps(out)) +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload-memory.json") + args = parser.parse_args() + check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads") + + print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}") + print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...") + large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)] + pixels = [(images.png(8000, 5000), "image/png")] + jpeg = [(images.sample_jpeg(), "image/jpeg")] + print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes") + + runs = [ + scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency), + scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency), + scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency), + ] + naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1]) + print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) " + f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB") + limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split() + result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20, + "api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive} + (ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"]) + print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB " + f"(limit {result['api_limit_mib']} MiB); results in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/worker_probe.py b/student-tests/worker_probe.py new file mode 100644 index 0000000..7b75584 --- /dev/null +++ b/student-tests/worker_probe.py @@ -0,0 +1,148 @@ +"""Пробник, который выполняется ВНУТРИ контейнера worker (там Pillow, модули приложения, доступ к БД и S3). + +Тест передаёт этот файл через stdin: `docker compose exec -T worker python - <команда> [аргументы]`. +Секреты остаются в контейнере; наружу выходит одна строка JSON (без ключей доступа, только факты). + + make — тестовое изображение, base64 (kind: thumb320, exif6, alpha, tiny, large, strip, big40) + inspect — метаданные из БД + декодированные объекты из S3: размеры, формат, EXIF, + цвета контрольных точек, SHA-256, список ключей под attachments// + exists — есть ли объект в bucket + imaging-limits — imaging.py на 40-Мп PNG: пик RSS дочернего процесса и отказ по лимиту пикселей +""" +import base64 +import hashlib +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +from uuid import UUID + +from PIL import Image + +import psycopg +from psycopg.rows import dict_row + +import settings +import storage + + +def make(kind: str) -> bytes: + out = io.BytesIO() + if kind == "thumb320": # 320x200 -> thumbnail 256x160 (контракт) + img = Image.new("RGB", (320, 200), (200, 30, 30)) + img.paste((30, 30, 200), (0, 0, 160, 200)) + img.save(out, "PNG") + elif kind == "exif6": # JPEG 400x300 с EXIF Orientation=6 и Make/Model/GPS + img = Image.new("RGB", (400, 300), (0, 160, 0)) + img.paste((255, 0, 0), (0, 0, 100, 300)) # левая полоса красная: после поворота на 90° по часовой — сверху + exif = Image.Exif() + exif[0x0112] = 6 + exif[0x010F] = "ProbeCamera" + exif[0x0110] = "Model-X" + exif[0x8825] = {1: "N", 2: (55.0, 45.0, 0.0)} + img.save(out, "JPEG", quality=95, exif=exif.tobytes()) + elif kind == "alpha": # RGBA 300x200: левая половина прозрачная + img = Image.new("RGBA", (300, 200), (0, 0, 255, 255)) + img.paste((0, 0, 0, 0), (0, 0, 150, 200)) + img.save(out, "PNG") + elif kind == "tiny": # 40x30: без увеличения для thumbnail/optimized + Image.new("RGB", (40, 30), (10, 120, 220)).save(out, "PNG") + elif kind == "large": # 3000x2000 JPEG: optimized 1280x853, thumbnail 256x170 + img = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB") + img.save(out, "JPEG", quality=90) + elif kind == "strip": # 5000x2: thumbnail 256x1 (минимум 1 px) + Image.new("RGB", (5000, 2), (90, 90, 90)).save(out, "PNG") + elif kind == "big40": # 8000x5000 = 40 Мп ровно на лимите + Image.new("RGB", (8000, 5000), (120, 60, 30)).save(out, "PNG") + else: + raise SystemExit("unknown kind") + return out.getvalue() + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row) + + +def read(key: str) -> bytes: + stream = storage.open_read(key) + try: + return b"".join(storage.iter_chunks(stream)) + finally: + stream.close() + + +def inspect(attachment_id: str) -> dict: + with connect() as conn: + a = conn.execute("SELECT state, object_key, source_key, pipeline_version, sha256, size_bytes " + "FROM attachments WHERE id = %s", (UUID(attachment_id),)).fetchone() + variants = conn.execute("SELECT name, content_type, width, height, sha256, size_bytes, object_key " + "FROM attachment_variants WHERE attachment_id = %s ORDER BY name", + (UUID(attachment_id),)).fetchall() + result = {"state": a["state"], "staging_present": a["source_key"] is not None and storage.exists(a["source_key"]), + "variant_rows": len(variants), "variants": {}} + if a["object_key"]: + original = read(a["object_key"]) + result["original_sha_ok"] = hashlib.sha256(original).hexdigest() == a["sha256"] + result["original_size_ok"] = len(original) == a["size_bytes"] + for v in variants: + data = read(v["object_key"]) + img = Image.open(io.BytesIO(data)) + info = {"format": img.format, "size": list(img.size), "mode": img.mode, + "meta_size_ok": [v["width"], v["height"]] == list(img.size), + "sha_ok": hashlib.sha256(data).hexdigest() == v["sha256"] and len(data) == v["size_bytes"], + "exif_tags": len(img.getexif()), "has_xmp": "xmp" in img.info or b"http://ns.adobe.com/xap" in data, + "quality_hint": None} + rgb = img.convert("RGBA") + w, h = rgb.size + info["pixel_top_center"] = list(rgb.getpixel((w // 2, 0))) + info["pixel_left_center"] = list(rgb.getpixel((0, h // 2))) + info["pixel_right_center"] = list(rgb.getpixel((w - 1, h // 2))) + result["variants"][v["name"]] = info + prefix = f"attachments/{attachment_id}/" + result["final_keys"] = sorted(key[len(prefix):] for key, _m, _s in storage.list_objects(prefix)) + return result + + +def imaging_limits() -> dict: + """Пик памяти процесса обработки на худшем допустимом входе и отказ при превышении лимита пикселей.""" + here = os.path.dirname(os.path.abspath(storage.__file__)) + report = {} + with tempfile.TemporaryDirectory(dir=settings.WORKER_TMP_DIR) as tmp: + source = os.path.join(tmp, "big.png") + with open(source, "wb") as f: + f.write(make("big40")) + env = {"PATH": os.environ.get("PATH", ""), "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for label, max_pixels in (("at_limit_40mpx", 40_000_000), ("over_limit_max_10mpx", 10_000_000)): + out = os.path.join(tmp, label) + os.mkdir(out) + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + proc = subprocess.run([sys.executable, os.path.join(here, "imaging.py"), source, out, "image/png", + str(max_pixels)], env=env, capture_output=True, timeout=120) + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + report[label] = {"exit_code": proc.returncode, "peak_rss_mib_max_children": round(peak / 1024, 1), + "rss_grew": peak > before} + report["memory_limit_mib"] = settings.WORKER_MEMORY_LIMIT_MB + report["timeout_s"] = settings.JOB_TIMEOUT_SECONDS + return report + + +def main(): + command = sys.argv[1] + if command == "make": + print(json.dumps({"b64": base64.b64encode(make(sys.argv[2])).decode("ascii")})) + elif command == "inspect": + print(json.dumps(inspect(sys.argv[2]))) + elif command == "exists": + print(json.dumps({"exists": storage.exists(sys.argv[2])})) + elif command == "imaging-limits": + print(json.dumps(imaging_limits())) + else: + raise SystemExit("unknown command") + + +if __name__ == "__main__": + main() diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..686f3f9 --- /dev/null +++ b/telemetry.py @@ -0,0 +1,450 @@ +"""Наблюдаемость (lab6): OpenTelemetry SDK для api, worker, dispatcher, janitor и ops. + +Все три сигнала уходят по OTLP/HTTP (protobuf) в OTel Collector (OTEL_EXPORTER_OTLP_ENDPOINT): + traces -> Collector -> Tempo + metrics -> Collector -> Prometheus (встроенный OTLP receiver) + logs -> Collector -> Loki (OTLP endpoint); те же записи пишутся в stderr одной JSON-строкой + (docker compose logs работает и без стека наблюдаемости). + +Ресурс (одинаковый у всех сигналов процесса): service.name (messenger-api / messenger-worker / +messenger-dispatcher / messenger-janitor), service.version (APP_VERSION), service.instance.id +(hostname контейнера = X-Instance-Id), deployment.environment.name=lab (OTEL_RESOURCE_ATTRIBUTES). + +Память при недоступном Collector или хранилище ограничена (экспорт идёт в фоновых потоках SDK, +запрос пользователя его не ждёт): + - спаны: BatchSpanProcessor, очередь OTEL_BSP_MAX_QUEUE_SIZE (2048); лишние отбрасываются и считаются + в otel.sdk.processor.span.processed{error.type="queue_full"} (OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED); + - логи: BatchLogRecordProcessor, очередь OTEL_BLRP_MAX_QUEUE_SIZE (2048), так же; + - метрики: агрегаты фиксированного размера (метки низкой кардинальности), экспорт раз в + OTEL_METRIC_EXPORT_INTERVAL; неудачный экспорт не накапливается, cumulative-счётчики догонят; + - каждый экспорт ограничен OTEL_EXPORTER_OTLP_TIMEOUT (5 с). + +Контекст между процессами — только W3C Trace Context (traceparent/tracestate); baggage не +распространяется, чтобы через заголовки не утекали произвольные данные. + +Чего НЕТ в телеметрии: паролей, Authorization, токенов сессий и их хешей, текстов сообщений, имён +файлов, байтов изображений, ключей и подписей S3. Метки метрик — только низкокардинальные: метод, +шаблон маршрута, класс/код статуса, тип и исход задания, имя очереди, состояние соединения. +В спанах и логах есть технические ID синтетических данных (job_id, attachment_id, request_id): +без них нельзя пройти от загрузки к результату worker; ID пользователя/чата туда не пишутся. +""" +import json +import logging +import os +import sys +import threading +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +from opentelemetry import context as otel_context +from opentelemetry import metrics, propagate, trace +from opentelemetry.trace import Link, SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +import settings + +SERVICE = "messenger" +VERSION = settings.APP_VERSION +_propagator = TraceContextTextMapPropagator() +propagate.set_global_textmap(_propagator) + +tracer = trace.get_tracer("messenger", VERSION) +meter = metrics.get_meter("messenger", VERSION) + +_state = {"configured": False, "providers": []} + +# Границы buckets (секунды) под ожидаемые задержки: p50/p95/p99 считаются из buckets в Prometheus. +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10) +JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120) +READY_BUCKETS = (0.5, 1, 2, 5, 10, 15, 30, 60, 120, 300, 600) +POOL_WAIT_BUCKETS = (0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5) + + +def enabled() -> bool: + return os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true" + + +# ---------- логи: JSON в stderr + OTLP ---------- + +_RESERVED = set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message", "asctime", "taskName"} +# Ключи, которые никогда не уходят ни в stderr, ни в OTLP (защита от случайного extra={...}). +_FORBIDDEN_KEYS = {"password", "authorization", "token", "secret", "text", "body", "filename"} + + +def _extra(record: logging.LogRecord) -> dict: + out = {} + for key, value in vars(record).items(): + if key in _RESERVED or key.startswith("_") or key.lower() in _FORBIDDEN_KEYS: + continue + if isinstance(value, (str, bool, int, float)): + out[key] = value + elif value is not None: + out[key] = str(value) + return out + + +class _EventFilter(logging.Filter): + """У каждой записи есть поле event (контракт): явное extra={"event": ...} или имя логгера.""" + + def filter(self, record): + if not getattr(record, "event", None): + record.event = record.name + return True + + +class _RateLimit(logging.Filter): + """Ошибки экспорта самого SDK (Collector недоступен) — не чаще раза в 30 с на источник.""" + + def __init__(self, seconds=30.0): + super().__init__() + self.seconds = seconds + self.last = {} + self.lock = threading.Lock() + + def filter(self, record): + if not record.name.startswith("opentelemetry"): + return True + key = (record.name, record.levelno) + now = time.monotonic() + with self.lock: + if now - self.last.get(key, -1e9) < self.seconds: + return False + self.last[key] = now + return True + + +class JsonFormatter(logging.Formatter): + """Одна JSON-строка: ts, severity, service.name, event, message, trace_id/span_id (если есть спан).""" + + def __init__(self, service_name: str): + super().__init__() + self.service_name = service_name + + def format(self, record): + data = { + "ts": datetime.fromtimestamp(record.created, timezone.utc).isoformat(timespec="milliseconds"), + "severity": record.levelname, + "service.name": self.service_name, + "event": getattr(record, "event", record.name), + "logger": record.name, + "message": record.getMessage(), + } + span = trace.get_current_span().get_span_context() + if span.is_valid: + data["trace_id"] = format(span.trace_id, "032x") + data["span_id"] = format(span.span_id, "016x") + data.update({k: v for k, v in _extra(record).items() if k != "event"}) + if record.exc_info and record.exc_info[0] is not None: + data["exception.type"] = record.exc_info[0].__name__ + data["exception.stacktrace"] = self.formatException(record.exc_info) + return json.dumps(data, ensure_ascii=False, default=str) + + +def _severity_map(): + from opentelemetry._logs import SeverityNumber + return {"DEBUG": SeverityNumber.DEBUG, "INFO": SeverityNumber.INFO, "WARNING": SeverityNumber.WARN, + "ERROR": SeverityNumber.ERROR, "CRITICAL": SeverityNumber.FATAL} + + +_SEVERITY = _severity_map() + + +class OtlpLogHandler(logging.Handler): + """logging -> OpenTelemetry Logs API. trace_id/span_id берутся из текущего контекста автоматически.""" + + def __init__(self, logger_provider): + super().__init__(level=logging.INFO) + self.provider = logger_provider + self.loggers = {} + + def emit(self, record): + if record.name.startswith("opentelemetry"): + return # ошибки экспорта не отправляем тем же экспортом (иначе петля при недоступном Collector) + try: + from opentelemetry._logs import LogRecord, SeverityNumber + + logger = self.loggers.get(record.name) + if logger is None: + logger = self.loggers[record.name] = self.provider.get_logger(record.name) + attributes = _extra(record) + attributes["code.function.name"] = record.funcName + attributes["code.line.number"] = record.lineno + if record.exc_info and record.exc_info[0] is not None: + attributes["exception.type"] = record.exc_info[0].__name__ + attributes["exception.stacktrace"] = logging.Formatter().formatException(record.exc_info) + level = {"WARNING": "WARN", "CRITICAL": "FATAL"}.get(record.levelname, record.levelname) + logger.emit(LogRecord( + timestamp=int(record.created * 1e9), + severity_text=level, + severity_number=_SEVERITY.get(record.levelname, SeverityNumber.INFO), + body=record.getMessage(), + attributes=attributes, + event_name=attributes.get("event"), + )) + except Exception: # noqa: BLE001 — сбой телеметрии не должен ломать логирование и запрос + self.handleError(record) + + +def _setup_logging(service_name: str, logger_provider=None) -> None: + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + root.setLevel(logging.INFO) + # stderr, как logging.basicConfig в lab5: stdout CLI (janitor --once, ops.py) остаётся чистым JSON-отчётом. + console = logging.StreamHandler(sys.stderr) + console.setFormatter(JsonFormatter(service_name)) + console.addFilter(_EventFilter()) + console.addFilter(_RateLimit()) + root.addHandler(console) + if logger_provider is not None: + otlp = OtlpLogHandler(logger_provider) + otlp.addFilter(_EventFilter()) + root.addHandler(otlp) + logging.getLogger("pika").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("opentelemetry").setLevel(logging.WARNING) + + +# ---------- сэмплирование ---------- + +def _sampler(): + """ParentBased(AlwaysOn), но клиентский спан БЕЗ родителя не записывается. + + Такие спаны — фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, janitor): + отдельная трасса на каждый SELECT только забивала бы Tempo. Всё, что относится к запросу или + заданию, имеет родителя (server/consumer/producer span) и сохраняется полностью. + """ + from opentelemetry.sdk.trace.sampling import (ALWAYS_ON, Decision, ParentBased, Sampler, + SamplingResult) + + class DropOrphanClientSpans(Sampler): + def __init__(self): + self.inner = ParentBased(ALWAYS_ON) + + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + parent = trace.get_current_span(parent_context).get_span_context() + if not parent.is_valid and kind == SpanKind.CLIENT: + return SamplingResult(Decision.DROP) + return self.inner.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state) + + def get_description(self): + return "ParentBased(AlwaysOn) without orphan client spans" + + return DropOrphanClientSpans() + + +# ---------- process / container ---------- + +def _read(path: str) -> str | None: + try: + return Path(path).read_text(encoding="ascii").strip() + except OSError: + return None + + +def _register_process_metrics() -> None: + from opentelemetry.metrics import CallbackOptions, Observation + + page = os.sysconf("SC_PAGE_SIZE") + + def cpu(_options: CallbackOptions): + times = os.times() + return [Observation(times.user, {"cpu.mode": "user"}), Observation(times.system, {"cpu.mode": "system"})] + + def memory(_options: CallbackOptions): + statm = _read("/proc/self/statm") + return [Observation(int(statm.split()[1]) * page)] if statm else [] + + meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU time of the process") + meter.create_observable_up_down_counter("process.memory.usage", [memory], unit="By", + description="Resident set size of the process") + + # cgroup v2 (Docker Desktop, современный Linux): контейнер целиком, включая дочерние процессы + # (у worker это imaging.py). Вне контейнера/на cgroup v1 этих метрик просто нет. + if _read("/sys/fs/cgroup/memory.current") is None: + return + + def container_memory(_options: CallbackOptions): + value = _read("/sys/fs/cgroup/memory.current") + return [Observation(int(value))] if value else [] + + def container_memory_limit(_options: CallbackOptions): + value = _read("/sys/fs/cgroup/memory.max") + return [Observation(int(value))] if value and value != "max" else [] + + def container_cpu(_options: CallbackOptions): + stat = _read("/sys/fs/cgroup/cpu.stat") or "" + for line in stat.splitlines(): + name, _, value = line.partition(" ") + if name == "usage_usec": + return [Observation(int(value) / 1e6)] + return [] + + def container_cpu_limit(_options: CallbackOptions): + value = (_read("/sys/fs/cgroup/cpu.max") or "").split() + if len(value) == 2 and value[0] != "max": + return [Observation(int(value[0]) / int(value[1]))] + return [] + + meter.create_observable_up_down_counter("container.memory.usage", [container_memory], unit="By", + description="Memory of the container cgroup") + meter.create_observable_up_down_counter("container.memory.limit", [container_memory_limit], unit="By", + description="Memory limit of the container cgroup") + meter.create_observable_counter("container.cpu.time", [container_cpu], unit="s", + description="CPU time of the container cgroup") + meter.create_observable_up_down_counter("container.cpu.limit", [container_cpu_limit], unit="{cpu}", + description="CPU limit of the container (cores)") + + +# ---------- настройка ---------- + +def setup(service_name: str, *, instrument_db: bool = True) -> None: + """Вызывается один раз при старте процесса, ДО открытия соединений с БД.""" + if _state["configured"]: + return + _state["configured"] = True + if not enabled(): + _setup_logging(service_name) + logging.getLogger("messenger.telemetry").info("OpenTelemetry disabled (OTEL_SDK_DISABLED=true)", + extra={"event": "telemetry.disabled"}) + return + + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab) читает Resource.create; явные атрибуты + # важнее переменных окружения: janitor и ops.py с общим .env не станут «messenger-api». + resource = Resource.create({ + "service.name": service_name, + "service.version": VERSION, + "service.instance.id": settings.INSTANCE_ID, + }) + + views = [ + View(instrument_name="http.server.request.duration", + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + View(instrument_name="db.client.connection.wait_time", + aggregation=ExplicitBucketHistogramAggregation(POOL_WAIT_BUCKETS)), + ] + interval = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "10000")) + meter_provider = MeterProvider( + resource=resource, views=views, + metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(), export_interval_millis=interval)], + ) + metrics.set_meter_provider(meter_provider) + + tracer_provider = TracerProvider(resource=resource, sampler=_sampler()) + tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(), meter_provider=meter_provider)) + trace.set_tracer_provider(tracer_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(), + meter_provider=meter_provider)) + set_logger_provider(logger_provider) + _state["providers"] = [tracer_provider, logger_provider, meter_provider] + + _setup_logging(service_name, logger_provider) + _register_process_metrics() + if instrument_db: + # Спаны PostgreSQL: один на запрос, текст SQL с плейсхолдерами; значения параметров НЕ пишутся + # (capture_parameters=False), sqlcommenter выключен. + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + PsycopgInstrumentor().instrument(tracer_provider=tracer_provider, enable_commenter=False, + capture_parameters=False, skip_dep_check=True) + logging.getLogger("messenger.telemetry").info( + "OpenTelemetry enabled: OTLP %s, service %s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "default"), + service_name, extra={"event": "telemetry.enabled"}) + + +def shutdown() -> None: + """Дослать буферы при штатной остановке (не дольше таймаута экспорта).""" + for provider in _state["providers"]: + try: + provider.shutdown() + except Exception: # noqa: BLE001 + pass + _state["providers"] = [] + + +# ---------- контекст W3C ---------- + +def inject(carrier: dict | None = None) -> dict: + """traceparent/tracestate текущего спана в carrier (заголовки AMQP, поле сообщения).""" + carrier = {} if carrier is None else carrier + _propagator.inject(carrier) + return carrier + + +def extract(carrier) -> otel_context.Context: + """Контекст из входящих заголовков (HTTP, AMQP headers, поле traceparent сообщения).""" + return _propagator.extract(carrier or {}) + + +def current_traceparent() -> str | None: + return inject().get("traceparent") + + +def context_from(traceparent: str | None, tracestate: str | None = None) -> otel_context.Context: + carrier = {"traceparent": traceparent} if traceparent else {} + if traceparent and tracestate: + carrier["tracestate"] = tracestate + return extract(carrier) + + +def link_to(traceparent: str | None) -> list: + """Link на другой контекст (например, исходная загрузка при replay оператора).""" + span_context = trace.get_current_span(context_from(traceparent)).get_span_context() + return [Link(span_context)] if span_context.is_valid else [] + + +def trace_id_hex() -> str | None: + span = trace.get_current_span().get_span_context() + return format(span.trace_id, "032x") if span.is_valid else None + + +# ---------- помощники для спанов ---------- + +def mark_error(span, error_type: str, message: str | None = None) -> None: + """Статус ERROR + error.type (низкокардинальный код). Текст — только безопасный код/класс.""" + span.set_attribute("error.type", error_type) + span.set_status(Status(StatusCode.ERROR, message or error_type)) + + +@contextmanager +def client_span(name: str, attributes: dict | None = None): + """Клиентский спан (S3, Redis). Исключение -> ERROR с типом исключения; само исключение не глотается.""" + with tracer.start_as_current_span(name, kind=SpanKind.CLIENT, attributes=attributes or {}, + record_exception=False, set_status_on_exception=False) as span: + try: + yield span + except Exception as exc: + mark_error(span, type(exc).__name__) + raise + + +@contextmanager +def internal_span(name: str, attributes: dict | None = None): + with tracer.start_as_current_span(name, kind=SpanKind.INTERNAL, attributes=attributes or {}, + record_exception=False, set_status_on_exception=False) as span: + try: + yield span + except Exception as exc: + mark_error(span, type(exc).__name__) + raise diff --git a/uploads.py b/uploads.py new file mode 100644 index 0000000..c01bfc1 --- /dev/null +++ b/uploads.py @@ -0,0 +1,519 @@ +"""Приём и проверка загружаемого изображения с ограниченной памятью. + +1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше + MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном + файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. +2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях + читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково + (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), + JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. +""" +import asyncio +import hashlib +import logging +import struct +import tempfile +import threading +import time +import unicodedata +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image +from starlette.requests import ClientDisconnect, Request + +import settings +from errors import bad_request, payload_too_large, unsupported_media_type + +log = logging.getLogger("messenger.uploads") + +PNG = "image/png" +JPEG = "image/jpeg" +ALLOWED_TYPES = (PNG, JPEG) +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + +SPOOL_IN_MEMORY = 256 * 1024 +INFLATE_CHUNK = 64 * 1024 +CRC_CHUNK = 64 * 1024 + +# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). +Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS + +# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. +_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) + + +@dataclass +class Upload: + file: BinaryIO + size: int + sha256: str + + +@dataclass +class ImageInfo: + content_type: str + width: int + height: int + + +class Damaged(Exception): + pass + + +class TooManyPixels(Exception): + pass + + +# ---------- имя файла и заголовки ---------- + +def validate_filename(value: str | None) -> str: + """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" + if value is None: + raise bad_request("Query parameter filename is required") + if not 1 <= len(value) <= 255 or not value.strip(): + raise bad_request("filename must be 1-255 characters") + if value in (".", ".."): + raise bad_request("filename is not allowed") + for char in value: + if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): + raise bad_request("filename contains forbidden characters") + return value + + +def declared_type(header: str | None) -> str: + base = (header or "").split(";", 1)[0].strip().lower() + if base not in ALLOWED_TYPES: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return base + + +def declared_length(header: str | None) -> int | None: + if header is None: + return None + if not header.isdigit(): + raise bad_request("Invalid Content-Length") + length = int(header) + if length > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + return length + + +# ---------- приём тела ---------- + +async def receive(request: Request) -> Upload: + """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" + spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS + stream = request.stream().__aiter__() + try: + while True: + wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) + if wait <= 0: + raise bad_request("Upload took too long") + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + raise bad_request("Upload stalled or took too long") from None + if not chunk: + continue + size += len(chunk) + if size > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + digest.update(chunk) + spool.write(chunk) + except ClientDisconnect: + spool.close() + log.info("Upload aborted by the client after %s bytes, nothing stored", size) + raise bad_request("Upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad_request("Empty body") + return Upload(spool, size, digest.hexdigest()) + + +async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: + """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. + + Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. + """ + read = 0 + deadline = time.monotonic() + seconds + try: + stream = request.stream().__aiter__() + while read <= limit: + wait = deadline - time.monotonic() + if wait <= 0: + return + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except (StopAsyncIteration, asyncio.TimeoutError): + return + read += len(chunk) + except (ClientDisconnect, RuntimeError): + return + + +# ---------- проверка содержимого ---------- + +def sniff(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return PNG + if head.startswith(JPEG_SIGNATURE): + return JPEG + return None + + +def inspect(upload: Upload, declared: str) -> ImageInfo: + upload.file.seek(0) + detected = sniff(upload.file.read(16)) + if detected is None: + raise unsupported_media_type("File is not a PNG or JPEG image") + if detected != declared: + raise unsupported_media_type("Content-Type does not match the file content") + upload.file.seek(0) + try: + with _decode_slots: + if detected == PNG: + width, height = check_png(upload.file) + upload.file.seek(0) + check_png_metadata(upload.file) + else: + width, height = check_jpeg(upload.file) + except TooManyPixels: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except Damaged as exc: + log.info("Rejected damaged %s: %s", detected, exc) + raise unsupported_media_type("Image is damaged or not supported") from None + except Image.DecompressionBombError: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: + log.info("Rejected damaged %s: %s", detected, type(exc).__name__) + raise unsupported_media_type("Image is damaged or not supported") from None + finally: + upload.file.seek(0) + return ImageInfo(detected, width, height) + + +def check_pixels(width: int, height: int) -> None: + if width < 1 or height < 1: + raise Damaged("zero dimension") + if width * height > settings.MAX_IMAGE_PIXELS: + raise TooManyPixels() + + +def check_jpeg(file: BinaryIO) -> tuple[int, int]: + with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS + width, height = image.size + check_pixels(width, height) + # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: + # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + return width, height + + +# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +FILTER_BYTES = bytes(range(5)) + + +class PngRows: + """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" + + def __init__(self, width, height, depth, color, interlace): + bits = depth * PNG_CHANNELS[color] + + def row_bytes(w): + return (w * bits + 7) // 8 + + passes = [(width, height)] if interlace == 0 else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + # (длина строки с байтом фильтра, байт в проходе) + self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] + self.expected = sum(total for _, total in self.passes) + self.index, self.pos = 0, 0 + self.total = 0 + + def feed(self, data: bytes) -> None: + view = memoryview(data) + self.total += len(data) + if self.total > self.expected: + raise Damaged("more pixel data than the header says") + offset = 0 + while offset < len(view): + stride, size = self.passes[self.index] + take = min(len(view) - offset, size - self.pos) + first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride + if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): + raise Damaged("invalid filter type") + self.pos += take + offset += take + if self.pos == size: + self.index, self.pos = self.index + 1, 0 + + def done(self) -> bool: + return self.total == self.expected + + +# Чанки, содержимое которых проверяется (держим в памяти; размер ограничен спецификацией): +# IHDR — 13 байт, PLTE — до 256 x 3 = 768, tRNS — до 256 (палитра) / 6 (RGB). +SMALL_CHUNK_LIMIT = {b"IHDR": 13, b"PLTE": 768, b"tRNS": 256} +CRITICAL = (b"IHDR", b"PLTE", b"IDAT", b"IEND") +# После данных изображения допустимы только время и текст (строгий список). +AFTER_IDAT = {b"IEND", b"tIME", b"tEXt", b"zTXt", b"iTXt"} +TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") +# Анимированные PNG (APNG) не поддерживаются: кадры — отдельная сущность, превью строится по одному кадру. +APNG_CHUNKS = (b"acTL", b"fcTL", b"fdAT") +# Как PngImagePlugin.MAX_TEXT_CHUNK в Pillow: больше 1 MiB распакованного текста в одном чанке — отказ. +TEXT_LIMIT = 1024 * 1024 +# Чанки метаданных до IDAT Pillow (Image.open) читает целиком: ограничиваем, чтобы память не росла. +# ICC-профили и EXIF реальных картинок — единицы-сотни KiB. +PRE_IDAT_CHUNK_LIMIT = 2 * 1024 * 1024 +# Чанки, которые по спецификации обязаны стоять ДО первого IDAT (PNG 3rd ed., 5.6 «Chunk ordering»). +# После IDAT допустимы только tIME, tEXt, zTXt, iTXt, кадры APNG и частные чанки. +BEFORE_IDAT = {b"PLTE", b"tRNS", b"cHRM", b"cICP", b"gAMA", b"iCCP", b"mDCV", b"cLLI", b"sBIT", b"sRGB", + b"bKGD", b"hIST", b"pHYs", b"sPLT", b"eXIf"} + + +def check_png(file: BinaryIO) -> tuple[int, int]: + """Структура PNG по спецификации (PNG 3rd ed., гл. 5 и 11) без декодирования пикселей в память. + + Кроме CRC каждого чанка проверяется СМЫСЛ критичных чанков и tRNS: CRC защищает только от + случайной порчи, а намеренно собранный файл с верными CRC может быть некорректным. + """ + if file.read(8) != PNG_SIGNATURE: + raise Damaged("signature") + rows = inflater = None + width = height = depth = 0 + color = None + palette_entries = 0 + seen = set() # какие чанки уже встречались + idat_started = idat_finished = False + first = True + while True: + header = file.read(8) + if len(header) < 8: + raise Damaged("truncated: no IEND") + length, kind = struct.unpack(">I4s", header) + if length > 0x7FFFFFFF: + raise Damaged("chunk length") + if not kind.isalpha() or not kind.isascii(): + raise Damaged("chunk type") + if first and kind != b"IHDR": + raise Damaged("IHDR must be first") + if kind[:1].isupper() and kind not in CRITICAL: + raise Damaged(f"unknown critical chunk {kind!r}") # такой чанк декодер обязан отвергнуть + if kind in SMALL_CHUNK_LIMIT and length > SMALL_CHUNK_LIMIT[kind]: + raise Damaged(f"{kind!r} is too long") + if kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT chunks must be consecutive") + idat_started = True + elif idat_started: + idat_finished = True + if kind in APNG_CHUNKS: + raise Damaged("animated PNG is not supported") + if kind in BEFORE_IDAT and idat_started: + raise Damaged(f"{kind!r} after IDAT") + if idat_finished and kind not in AFTER_IDAT: + raise Damaged(f"{kind!r} after IDAT") + if not idat_started and kind != b"IDAT" and length > PRE_IDAT_CHUNK_LIMIT: + raise Damaged(f"{kind!r} metadata chunk is too large") + if kind == b"tIME" and length != 7: + raise Damaged("tIME size") + if kind in (b"IHDR", b"PLTE", b"tRNS", b"IEND") and kind in seen: + raise Damaged(f"duplicate {kind!r}") + seen.add(kind) + + crc = zlib.crc32(kind) + kept = b"" + text = TextChunk(kind) if kind in TEXT_CHUNKS else None + left = length + while left: + piece = file.read(min(CRC_CHUNK, left)) + if not piece: + raise Damaged("truncated chunk") + left -= len(piece) + crc = zlib.crc32(piece, crc) + if kind in SMALL_CHUNK_LIMIT: + kept += piece + elif kind == b"IDAT": + feed_inflate(inflater, rows, piece) + elif text is not None: + text.feed(piece) + if text is not None: + text.finish() + stored = file.read(4) + if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: + raise Damaged(f"CRC mismatch in {kind!r}") + + if kind == b"IHDR": + if length != 13: + raise Damaged("IHDR") + width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) + if width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise Damaged("dimensions") + check_pixels(width, height) # до распаковки + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: + raise Damaged("color type / bit depth") + if compression != 0 or filter_method != 0 or interlace not in (0, 1): + raise Damaged("IHDR methods") + rows = PngRows(width, height, depth, color, interlace) + inflater = zlib.decompressobj() + elif kind == b"PLTE": + # 1-256 записей по 3 байта; для палитровых — не больше 2^глубина; у серых палитры быть не может. + if color in (0, 4): + raise Damaged("PLTE in a grayscale image") + if length == 0 or length % 3: + raise Damaged("palette size") + palette_entries = length // 3 + if color == 3 and palette_entries > 1 << depth: + raise Damaged("palette larger than the bit depth allows") + elif kind == b"tRNS": + # Прозрачность: серое — 2 байта, RGB — 6, палитра — не больше записей PLTE; при альфа-канале запрещён. + if color in (4, 6): + raise Damaged("tRNS with an alpha channel") + if color == 3 and not palette_entries: + raise Damaged("tRNS before PLTE") + expected_ok = {0: length == 2, 2: length == 6, 3: 1 <= length <= palette_entries}[color] + if not expected_ok: + raise Damaged("tRNS size") + elif kind == b"IEND": + if length != 0: + raise Damaged("IEND with data") + break + first = False + + if file.read(1): + raise Damaged("data after IEND") + if color == 3 and not palette_entries: + raise Damaged("palette image without PLTE") + if not idat_started: + raise Damaged("no IDAT") + tail = inflater.flush(INFLATE_CHUNK) + if tail: + rows.feed(tail) + if not inflater.eof or inflater.unused_data or not rows.done(): + raise Damaged("pixel data is incomplete") + return width, height + + +def check_png_metadata(file: BinaryIO) -> None: + """Второе мнение: стандартный парсер Pillow разбирает метаданные до IDAT (gAMA, iCCP, pHYs, tEXt…), + НЕ читая пиксели. Только open, без verify(): verify читает каждый чанк, включая IDAT до 10 MiB, + целиком в память, а CRC и данные после IDAT уже потоково проверил check_png. + Нужен для вспомогательных чанков: по спецификации их можно игнорировать, но если Pillow + не может их разобрать, файл нельзя будет обработать (превью в lab5) — лучше отклонить сразу. + """ + with Image.open(file, formats=["PNG"]): + pass # open разбирает все чанки до первого IDAT; после IDAT check_png пускает только время и текст + + +class TextChunk: + """Потоковая проверка tEXt / zTXt / iTXt: ключ 1-79 байт, метод сжатия 0, сжатый текст — + корректный zlib не больше TEXT_LIMIT после распаковки (распаковка кусками, текст не хранится).""" + + def __init__(self, kind: bytes): + self.kind = kind + self.head = b"" + self.done_header = False + self.inflater = None + self.size = 0 + + def feed(self, piece: bytes) -> None: + if not self.done_header: + self.head += piece + rest = self._header() + if rest is None: + if len(self.head) > 4096: + raise Damaged("text chunk header") + return + piece = rest + if self.inflater is not None and piece: + self._inflate(piece) + + def _keyword_end(self) -> int | None: + end = self.head.find(b"\x00") + if end == -1: + return None + if not 1 <= end <= 79: + raise Damaged("text keyword") + return end + + def _header(self) -> bytes | None: + end = self._keyword_end() + if end is None: + return None + h = self.head + if self.kind == b"tEXt": + self.done_header = True + return b"" + if self.kind == b"zTXt": + if len(h) < end + 2: + return None + if h[end + 1] != 0: + raise Damaged("zTXt compression method") + self.done_header, self.inflater = True, zlib.decompressobj() + return h[end + 2:] + # iTXt: keyword \0 flag method language \0 translated-keyword \0 text + if len(h) < end + 3: + return None + flag, method = h[end + 1], h[end + 2] + if flag not in (0, 1) or (flag and method != 0): + raise Damaged("iTXt compression") + lang_end = h.find(b"\x00", end + 3) + tkey_end = h.find(b"\x00", lang_end + 1) if lang_end != -1 else -1 + if tkey_end == -1: + return None + self.done_header = True + if flag: + self.inflater = zlib.decompressobj() + return h[tkey_end + 1:] + + def _inflate(self, data: bytes) -> None: + if self.inflater.eof: + raise Damaged("data after compressed text") + out = self.inflater.decompress(data, INFLATE_CHUNK) + self.size += len(out) + while self.inflater.unconsumed_tail and not self.inflater.eof and self.size <= TEXT_LIMIT: + out = self.inflater.decompress(self.inflater.unconsumed_tail, INFLATE_CHUNK) + self.size += len(out) + if self.size > TEXT_LIMIT: + raise Damaged("compressed text is too large") + if self.inflater.eof and self.inflater.unused_data: + raise Damaged("data after compressed text") + + def finish(self) -> None: + if not self.done_header: + raise Damaged("text chunk header") + if self.inflater is not None: + self.size += len(self.inflater.flush(INFLATE_CHUNK)) + if not self.inflater.eof or self.size > TEXT_LIMIT: + raise Damaged("compressed text") + + +def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: + """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" + if inflater.eof: + if data: + raise Damaged("data after the end of zlib stream") + return + out = inflater.decompress(data, INFLATE_CHUNK) + rows.feed(out) + while inflater.unconsumed_tail and not inflater.eof: + out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) + rows.feed(out) + if inflater.eof and inflater.unused_data: + raise Damaged("data after the end of zlib stream") diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..0f58f51 --- /dev/null +++ b/worker.py @@ -0,0 +1,527 @@ +"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose). + +Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1): +одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди. + +Обработка одного сообщения: + 1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ. + 2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый + lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op. + 3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает -> + аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued. + Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения. + 4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД. + 5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь). + 6. PUT в конечный prefix: attachments//v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp. + Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи. + 7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT. + 8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД. + 9. Удаление исходника из staging (если не вышло — доделает janitor). +Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД, +таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed. +""" +import hashlib +import json +import logging +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path + +import psycopg +from opentelemetry import trace +from opentelemetry.trace import NonRecordingSpan, SpanKind +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-worker") + +import broker # noqa: E402 +import faults # noqa: E402 +import imaging # noqa: E402 +import pipeline # noqa: E402 +import storage # noqa: E402 + +log = logging.getLogger("messenger.worker") + +# lab6: метрики фоновой обработки (contracts/observability.md). Метки: тип задания, исход, очередь. +JOB_ATTRS = {"job.type": pipeline.JOB_TYPE} +JOB_DURATION = telemetry.meter.create_histogram( + "messenger.job.duration", unit="s", + description="Attempt of an attachment job from claim to result (outcome: ready/retry/failed/stale/lease_lost)") +TIME_TO_READY = telemetry.meter.create_histogram( + "messenger.attachment.time_to_ready", unit="s", + description="From durable acceptance (job COMMIT in API) to ready COMMIT in worker") +MESSAGES = telemetry.meter.create_counter( + "messenger.messages.consumed", unit="{message}", + description="Broker deliveries by outcome: processed (claimed), noop (duplicate/not due), dead_lettered, requeued") +BUSY = {"value": 0} +BUSY_LOCK = threading.Lock() + + +def _observe_busy(_options): + from opentelemetry.metrics import Observation + return [Observation(BUSY["value"], {"messaging.destination.name": pipeline.QUEUE})] + + +def _observe_capacity(_options): + from opentelemetry.metrics import Observation + return [Observation(settings.WORKER_CONCURRENCY, {"messaging.destination.name": pipeline.QUEUE})] + + +telemetry.meter.create_observable_up_down_counter( + "messenger.worker.busy", [_observe_busy], unit="{consumer}", description="Consumers processing a job now") +telemetry.meter.create_observable_up_down_counter( + "messenger.worker.capacity", [_observe_capacity], unit="{consumer}", + description="Consumers of this worker process (WORKER_CONCURRENCY)") + + +def busy(delta: int) -> None: + with BUSY_LOCK: + BUSY["value"] += delta + +STOP = threading.Event() +HERE = Path(__file__).resolve().parent +VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} +IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large", + imaging.EXIT_MEMORY: "resource_limit"} + + +class JobError(Exception): + def __init__(self, code: str, permanent: bool): + super().__init__(code) + self.code = code + self.permanent = permanent + + +class LeaseLost(Exception): + pass + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def final_prefix(claim: pipeline.Claim) -> str: + return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/" + + +class LeaseKeeper(threading.Thread): + """Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку.""" + + def __init__(self, claim: pipeline.Claim): + super().__init__(daemon=True) + self.claim = claim + self.done = threading.Event() + self.lost = threading.Event() + self.frozen = threading.Event() # fault worker_freeze: перестать продлевать + + def run(self): + interval = max(1.0, settings.JOB_LEASE_SECONDS / 3) + conn = None + while not self.done.wait(interval): + if self.frozen.is_set(): + continue + try: + if conn is None: + conn = connect_db() + if not pipeline.renew_lease(conn, self.claim): + log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id, + extra={"event": "job.lease_lost", "job.id": str(self.claim.job_id)}) + self.lost.set() + return + except psycopg.Error as exc: + # Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT). + log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__) + if conn is not None: + conn.close() + conn = None + if conn is not None: + conn.close() + + def check(self): + if self.lost.is_set(): + raise LeaseLost() + + +def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: + # Дочернему процессу — только лимиты, никаких секретов окружения. + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1", + "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for fault in ("hang", "invalid", "oom"): + if faults.hit("imaging_" + fault): + env["FAULT_IMAGING"] = fault + with telemetry.internal_span("imaging", {"messenger.pipeline.version": claim.pipeline_version, + "messenger.source.content_type": claim.content_type}) as span: + result = _run_imaging(env, source, out_dir, claim, keeper) + span.set_attribute("messenger.variants", len(result["variants"])) + return result + + +def _run_imaging(env: dict, source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: + process = subprocess.Popen( + [sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type, + str(settings.MAX_IMAGE_PIXELS)], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir), + ) + deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS + try: + while True: + try: + output, _ = process.communicate(timeout=0.5) + break + except subprocess.TimeoutExpired: + if time.monotonic() > deadline: + raise JobError("processing_timeout", permanent=False) from None + keeper.check() + finally: + if process.poll() is None: + process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку + process.wait() + code = process.returncode + if code in IMAGING_EXIT: + raise JobError(IMAGING_EXIT[code], permanent=True) + if code != 0: + # Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены. + raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False) + return json.loads(output.decode("utf-8").strip().splitlines()[-1]) + + +def sha256_file(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with open(path, "rb") as file: + while chunk := file.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None: + if faults.hit("worker_s3_error"): + raise JobError("storage_unavailable", permanent=False) + with open(path, "rb") as file: + try: + storage.put(key, file, size, content_type, sha256) + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + + +def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]: + """Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов.""" + source = workdir / "source" + try: + storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES) + except storage.ObjectMissing: + raise JobError("source_missing", permanent=True) from None + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + with telemetry.internal_span("verify source sha256"): + size, digest = sha256_file(source) + if size != claim.size_bytes or digest != claim.sha256: + # Staging не совпадает с тем, что принял API: публиковать нельзя. + raise JobError("source_corrupted", permanent=True) + keeper.check() + + out_dir = workdir / "out" + out_dir.mkdir() + result = run_imaging(source, out_dir, claim, keeper) + keeper.check() + + prefix = final_prefix(claim) + original_key = prefix + "original" + put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes) + variants = [] + for item in result["variants"]: + path = out_dir / item["file"] + v_size, v_sha = sha256_file(path) + key = prefix + VARIANT_KEYS[item["name"]] + put_file(key, path, item["content_type"], v_sha, v_size) + variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size, + "width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key}) + keeper.check() + return original_key, variants + + +@dataclass +class Outcome: + ack: bool = True # False -> nack с requeue (не смогли даже записать claim) + reject: bool = False # True -> reject без requeue -> DLQ + cleanup_key: str | None = None + attachment_id: object = None + span_context: object = None # спан обработки: очистка staging после ack — в той же трассе + delivery: str = "processed" # метка messenger.messages.consumed: processed / noop + + +def carrier_from(headers, body: bytes) -> dict: + """W3C контекст сообщения: native headers AMQP (основной carrier), иначе поле traceparent тела.""" + carrier = {} + for key in ("traceparent", "tracestate"): + value = (headers or {}).get(key) + if isinstance(value, bytes): + value = value.decode("ascii", "replace") + if isinstance(value, str): + carrier[key] = value + if "traceparent" not in carrier: + try: + message = json.loads(body.decode("utf-8")) + if isinstance(message, dict) and isinstance(message.get("traceparent"), str): + carrier["traceparent"] = message["traceparent"] + except (UnicodeDecodeError, ValueError): + pass + return carrier + + +class Consumer(threading.Thread): + def __init__(self, number: int): + super().__init__(name=f"consumer-{number}", daemon=True) + self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}" + self.db = None + + def database(self): + if self.db is None or self.db.closed: + self.db = connect_db() + return self.db + + def handle(self, body: bytes, redelivered: bool, headers=None) -> Outcome: + """Спан CONSUMER «process attachments.process.v1» — ребёнок спана publish из dispatcher.""" + attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "process", + "messaging.destination.name": pipeline.QUEUE, "messaging.consumer.group.name": pipeline.QUEUE, + "messaging.rabbitmq.message.redelivered": redelivered} + with telemetry.tracer.start_as_current_span( + f"process {pipeline.QUEUE}", context=telemetry.extract(carrier_from(headers, body)), + kind=SpanKind.CONSUMER, attributes=attributes, + record_exception=False, set_status_on_exception=False) as span: + outcome = self._handle(body, redelivered, span) + outcome.span_context = span.get_span_context() + kind = "dead_lettered" if outcome.reject else "requeued" if not outcome.ack else outcome.delivery + span.set_attribute("messenger.delivery", kind) + MESSAGES.add(1, {"messaging.destination.name": pipeline.QUEUE, "outcome": kind}) + return outcome + + def _handle(self, body: bytes, redelivered: bool, span) -> Outcome: + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + log.warning("invalid message -> DLQ (%s)", exc, extra={"event": "job.invalid_message"}) + telemetry.mark_error(span, "invalid_message") + return Outcome(reject=True) + span.set_attributes({"messaging.message.id": f"{message['job_id']}:{message['attempt']}", + "messenger.job.id": message["job_id"], + "messenger.attachment.id": message["attachment_id"], + "messenger.message.attempt": message["attempt"]}) + try: + result = pipeline.claim(self.database(), message, self.owner, redelivered) + except psycopg.Error as exc: + log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__, + extra={"event": "job.claim_failed", "job.id": message["job_id"], + "error.type": type(exc).__name__}) + telemetry.mark_error(span, type(exc).__name__) + self.db = None + STOP.wait(2) + return Outcome(ack=False) + if isinstance(result, pipeline.Reject): + log.warning("job %s: %s -> DLQ", message["job_id"], result.reason, + extra={"event": "job.rejected", "job.id": message["job_id"], "reason": result.reason}) + telemetry.mark_error(span, result.reason) + return Outcome(reject=True) + if isinstance(result, pipeline.Skip): + log.info("job %s: no-op (%s), ack", message["job_id"], result.reason, + extra={"event": "job.noop", "job.id": message["job_id"], "reason": result.reason}) + span.set_attribute("messenger.noop.reason", result.reason) + return Outcome(delivery="noop") + span.set_attribute("messenger.job.attempt", result.attempt) + busy(+1) + try: + return self.execute(result, span) + finally: + busy(-1) + + def execute(self, claim: pipeline.Claim, span) -> Outcome: + started = time.monotonic() + fields = {"job.id": str(claim.job_id), "attachment.id": str(claim.attachment_id), "attempt": claim.attempt} + log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt, + claim.max_attempts, claim.attachment_id, extra={"event": "job.claimed", **fields}) + keeper = LeaseKeeper(claim) + keeper.start() + workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR)) + outcome_name = "ready" + try: + try: + original_key, variants = process_job(claim, keeper, workdir) + faults.crash("worker_crash_after_put") + if faults.hit("worker_freeze"): + keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят + time.sleep(settings.JOB_LEASE_SECONDS * 3) + with telemetry.internal_span("commit result"): + time_to_ready = pipeline.finish_ready(self.database(), claim, original_key, variants, + self.owner) + if time_to_ready is None: + outcome_name = "stale" # аренду перехватили: fencing отклонил результат + except JobError as exc: + outcome_name = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner) + telemetry.mark_error(span, exc.code) + log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code, + "permanent" if exc.permanent else "temporary", outcome_name, + extra={"event": f"job.{outcome_name}", "error.type": exc.code, + "permanent": exc.permanent, **fields}) + return Outcome() + except LeaseLost: + outcome_name = "lease_lost" + telemetry.mark_error(span, "lease_lost") + log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt, + extra={"event": "job.lease_lost", **fields}) + return Outcome() + except psycopg.Error: + raise + except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены + log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt, + extra={"event": "job.unexpected_error", **fields}) + telemetry.mark_error(span, "processing_failed") + outcome_name = pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner) + return Outcome() + except psycopg.Error as exc: + # Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и + # dispatcher вернёт job в очередь; если прошёл — повтор увидит ready. + outcome_name = "db_error" + telemetry.mark_error(span, type(exc).__name__) + log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__, + extra={"event": "job.db_error", "error.type": type(exc).__name__, **fields}) + self.db = None + return Outcome() + finally: + keeper.done.set() + shutil.rmtree(workdir, ignore_errors=True) + elapsed = time.monotonic() - started + span.set_attribute("messenger.job.outcome", outcome_name) + JOB_DURATION.record(elapsed, JOB_ATTRS | {"outcome": outcome_name}) + if time_to_ready is None: + telemetry.mark_error(span, "stale_result_rejected") + log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id, + claim.attempt, extra={"event": "job.stale_result_rejected", **fields}) + return Outcome() + TIME_TO_READY.record(time_to_ready, JOB_ATTRS) + faults.crash("worker_crash_after_commit") + log.info("job %s attempt %s: ready in %.0f ms (%.1f s since upload)", claim.job_id, claim.attempt, + (time.monotonic() - started) * 1000, time_to_ready, + extra={"event": "job.ready", "duration_ms": round((time.monotonic() - started) * 1000, 1), + "time_to_ready_s": round(time_to_ready, 3), **fields}) + return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id) + + def cleanup_staging(self, outcome: Outcome) -> None: + """После COMMIT ready и ack: исходник в staging больше не нужен.""" + parent = trace.set_span_in_context(NonRecordingSpan(outcome.span_context)) if outcome.span_context else None + with telemetry.tracer.start_as_current_span("cleanup staging", context=parent): + try: + storage.delete(outcome.cleanup_key) + self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' " + "AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key)) + except (storage.StorageUnavailable, psycopg.Error) as exc: + log.info("staging cleanup postponed to janitor: %s", type(exc).__name__, + extra={"event": "staging.cleanup_postponed", "error.type": type(exc).__name__}) + self.db = None + + def consume_once(self) -> None: + connection = broker.connect() + try: + channel = connection.channel() + channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH) + broker.check_topology(channel) + log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH, + extra={"event": "worker.consuming"}) + for method, properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1): + if STOP.is_set(): + break + if method is None: + continue + box = {} + headers = properties.headers if properties is not None else None + runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle( + body, method.redelivered, headers)), daemon=True) + runner.start() + while runner.is_alive(): + connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа + outcome = box.get("outcome", Outcome(ack=False)) + if outcome.reject: + channel.basic_reject(method.delivery_tag, requeue=False) + elif outcome.ack: + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + if outcome.cleanup_key: + self.cleanup_staging(outcome) + try: + channel.cancel() + except broker.BrokerError: + pass + finally: + try: + connection.close() + except broker.BrokerError: + pass + + def safe_handle(self, body, redelivered, headers=None) -> Outcome: + try: + return self.handle(body, redelivered, headers) + except Exception: # noqa: BLE001 + log.exception("unexpected error while handling a message; requeue", + extra={"event": "job.handler_crashed"}) + self.db = None + STOP.wait(2) + return Outcome(ack=False) + + def run(self): + pause = 1.0 + while not STOP.is_set(): + try: + self.consume_once() + pause = 1.0 + except broker.BrokerError as exc: + log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name, + type(exc).__name__, pause, + extra={"event": "broker.unavailable", "error.type": type(exc).__name__}) + STOP.wait(pause) + pause = min(pause * 2, 15) + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True) + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)] + for consumer in consumers: + consumer.start() + log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, " + "fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH, + settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB, + "ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"}) + while not STOP.wait(2): + if not all(consumer.is_alive() for consumer in consumers): + log.error("a consumer thread died, exiting so that Docker restarts the worker") + os._exit(1) + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят. + for consumer in consumers: + consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10) + log.info("stopped", extra={"event": "service.stopped"}) + telemetry.shutdown() + + +if __name__ == "__main__": + main()