From 79ef8807d5d505a60a9eafd41d4a4d0f1dbbd187 Mon Sep 17 00:00:00 2001 From: kalibr456 <145665723+kalibr456@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:25:38 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A06:=20OpenTelemetry,=20Collector,=20?= =?UTF-8?q?Prometheus/Loki/Tempo,=20Grafana,=20SLO=20=D0=B8=20=D0=B0=D0=BB?= =?UTF-8?q?=D0=B5=D1=80=D1=82=D1=8B,=20=D0=B8=D0=BD=D1=86=D0=B8=D0=B4?= =?UTF-8?q?=D0=B5=D0=BD=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 50 +- .gitattributes | 11 + .github/pull_request_template.md | 28 +- Dockerfile | 31 + REPORT.md | 363 ++- attachments.py | 80 + auth.py | 150 ++ broker.py | 104 + compose.yaml | 494 ++++ db.py | 452 ++++ dispatcher.py | 260 ++ docs/runbooks.md | 123 + errors.py | 40 + evidence/attachments.txt | 96 + evidence/balancing.txt | 18 + evidence/check.txt | 6 + evidence/configs.txt | 32 + evidence/consistency.txt | 47 + evidence/dashboard.txt | 36 + evidence/db.txt | 21 + evidence/explain-history.txt | 109 + evidence/explain.txt | 46 + evidence/failover.txt | 33 + evidence/failures.txt | 29 + evidence/idempotency.txt | 22 + evidence/incident-collector-down.json | 39 + evidence/incident-collector.txt | 20 + evidence/incident-db-slow.json | 82 + evidence/incident-db.txt | 64 + evidence/incident-telemetry-down.json | 273 ++ evidence/incident-telemetry.txt | 47 + evidence/incident-worker-stop.json | 50 + evidence/incident-worker.txt | 49 + evidence/jobs-concurrency.txt | 8 + evidence/jobs-crash.txt | 15 + evidence/jobs-janitor-replay.txt | 13 + evidence/jobs-limits.txt | 11 + evidence/jobs-outbox.txt | 9 + evidence/jobs-recovery.txt | 16 + evidence/jobs-retry.txt | 15 + evidence/jobs-variants.txt | 14 + evidence/load.json | 17 + evidence/load.txt | 22 + evidence/native-sandbox/attachments.txt | 96 + evidence/native-sandbox/check.txt | 6 + evidence/native-sandbox/configs.txt | 28 + evidence/native-sandbox/dashboard.txt | 36 + evidence/native-sandbox/idempotency.txt | 22 + .../incident-collector-down.json | 39 + .../native-sandbox/incident-collector.txt | 22 + evidence/native-sandbox/incident-db-slow.json | 82 + evidence/native-sandbox/incident-db.txt | 56 + .../incident-telemetry-down.json | 307 +++ .../native-sandbox/incident-telemetry.txt | 41 + .../native-sandbox/incident-worker-stop.json | 49 + evidence/native-sandbox/incident-worker.txt | 43 + evidence/native-sandbox/jobs-concurrency.txt | 8 + evidence/native-sandbox/jobs-outbox.txt | 9 + evidence/native-sandbox/jobs-recovery.txt | 16 + evidence/native-sandbox/jobs-retry.txt | 15 + evidence/native-sandbox/jobs-variants.txt | 14 + evidence/native-sandbox/load.json | 17 + evidence/native-sandbox/load.txt | 22 + evidence/native-sandbox/race.txt | 10 + evidence/native-sandbox/smoke.txt | 6 + evidence/native-sandbox/student-tests.txt | 26 + evidence/native-sandbox/summary.md | 48 + evidence/native-sandbox/telemetry-history.txt | 9 + evidence/native-sandbox/telemetry.txt | 145 ++ evidence/persistence.txt | 40 + evidence/race.txt | 10 + evidence/redis.txt | 14 + evidence/s3-security.txt | 53 + evidence/smoke.txt | 6 + evidence/stack.txt | 66 + evidence/student-tests.txt | 26 + evidence/summary.md | 108 + evidence/telemetry-history.txt | 9 + evidence/telemetry.txt | 145 ++ evidence/upload-memory.json | 124 + evidence/upload-memory.txt | 13 + faults.py | 51 + imaging.py | 209 ++ infra/grafana/dashboards/messenger.json | 2288 +++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 13 + .../provisioning/datasources/datasources.yaml | 67 + infra/haproxy.cfg | 77 + infra/loki/loki.yaml | 52 + infra/otel/collector.yaml | 129 + infra/prometheus/prometheus.yml | 42 + infra/prometheus/rules/alerts.yml | 113 + infra/prometheus/rules/recording.yml | 50 + infra/prometheus/tests/alerts_test.yml | 121 + infra/rabbitmq.conf | 22 + infra/redis.conf | 13 + infra/tempo/tempo.yaml | 39 + janitor.py | 322 +++ main.py | 666 +++++ migrate.py | 8 + migrations/001_init.sql | 33 + migrations/002_text_as_bytes.sql | 11 + migrations/003_idempotency.sql | 11 + migrations/004_attachments.sql | 48 + migrations/005_jobs.sql | 133 + ops.py | 201 ++ pipeline.py | 502 ++++ requirements.txt | 49 + s3_init.py | 103 + scripts/build_dashboard.py | 265 ++ scripts/init_secrets.py | 157 ++ scripts/restart.cmd | 15 + scripts/restart.sh | 14 + scripts/stop-one.cmd | 6 + scripts/stop-one.sh | 5 + scripts/stop_one.py | 29 + scripts/worker.cmd | 8 + scripts/worker.sh | 11 + settings.py | 111 + storage.py | 209 ++ student-tests/check_configs.py | 107 + student-tests/check_dashboard.py | 119 + student-tests/check_stack.py | 118 + student-tests/collect_evidence.py | 449 ++++ student-tests/common.py | 347 +++ student-tests/demo_instances.py | 16 + student-tests/explain.py | 28 + student-tests/explain.sql | 76 + student-tests/failover_metrics.py | 129 + student-tests/fixtures/progressive.jpg | Bin 0 -> 1081 bytes student-tests/fixtures/sample.jpg | Bin 0 -> 1492 bytes student-tests/images.py | 93 + student-tests/incidents.py | 499 ++++ student-tests/loadgen.py | 102 + student-tests/obs.py | 223 ++ student-tests/redis_info.py | 29 + student-tests/rolling_update.py | 88 + student-tests/run_failover.py | 55 + student-tests/run_persistence.py | 51 + student-tests/test_api.py | 258 ++ student-tests/test_attachments.py | 338 +++ student-tests/test_balancing.py | 103 + student-tests/test_consistency.py | 276 ++ student-tests/test_db.py | 101 + student-tests/test_expiry.py | 57 + student-tests/test_failures.py | 110 + student-tests/test_idempotency.py | 174 ++ student-tests/test_janitor_replay.py | 154 ++ student-tests/test_jobs.py | 567 ++++ student-tests/test_race.py | 57 + student-tests/test_s3_security.py | 146 ++ student-tests/test_telemetry.py | 258 ++ student-tests/upload_memory.py | 222 ++ student-tests/worker_probe.py | 148 ++ telemetry.py | 450 ++++ uploads.py | 519 ++++ worker.py | 527 ++++ 156 files changed, 18393 insertions(+), 35 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 attachments.py create mode 100644 auth.py create mode 100644 broker.py create mode 100644 compose.yaml create mode 100644 db.py create mode 100644 dispatcher.py create mode 100644 docs/runbooks.md create mode 100644 errors.py create mode 100644 evidence/attachments.txt create mode 100644 evidence/balancing.txt create mode 100644 evidence/check.txt create mode 100644 evidence/configs.txt create mode 100644 evidence/consistency.txt create mode 100644 evidence/dashboard.txt create mode 100644 evidence/db.txt create mode 100644 evidence/explain-history.txt create mode 100644 evidence/explain.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failures.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/incident-collector-down.json create mode 100644 evidence/incident-collector.txt create mode 100644 evidence/incident-db-slow.json create mode 100644 evidence/incident-db.txt create mode 100644 evidence/incident-telemetry-down.json create mode 100644 evidence/incident-telemetry.txt create mode 100644 evidence/incident-worker-stop.json create mode 100644 evidence/incident-worker.txt create mode 100644 evidence/jobs-concurrency.txt create mode 100644 evidence/jobs-crash.txt create mode 100644 evidence/jobs-janitor-replay.txt create mode 100644 evidence/jobs-limits.txt create mode 100644 evidence/jobs-outbox.txt create mode 100644 evidence/jobs-recovery.txt create mode 100644 evidence/jobs-retry.txt create mode 100644 evidence/jobs-variants.txt create mode 100644 evidence/load.json create mode 100644 evidence/load.txt create mode 100644 evidence/native-sandbox/attachments.txt create mode 100644 evidence/native-sandbox/check.txt create mode 100644 evidence/native-sandbox/configs.txt create mode 100644 evidence/native-sandbox/dashboard.txt create mode 100644 evidence/native-sandbox/idempotency.txt create mode 100644 evidence/native-sandbox/incident-collector-down.json create mode 100644 evidence/native-sandbox/incident-collector.txt create mode 100644 evidence/native-sandbox/incident-db-slow.json create mode 100644 evidence/native-sandbox/incident-db.txt create mode 100644 evidence/native-sandbox/incident-telemetry-down.json create mode 100644 evidence/native-sandbox/incident-telemetry.txt create mode 100644 evidence/native-sandbox/incident-worker-stop.json create mode 100644 evidence/native-sandbox/incident-worker.txt create mode 100644 evidence/native-sandbox/jobs-concurrency.txt create mode 100644 evidence/native-sandbox/jobs-outbox.txt create mode 100644 evidence/native-sandbox/jobs-recovery.txt create mode 100644 evidence/native-sandbox/jobs-retry.txt create mode 100644 evidence/native-sandbox/jobs-variants.txt create mode 100644 evidence/native-sandbox/load.json create mode 100644 evidence/native-sandbox/load.txt create mode 100644 evidence/native-sandbox/race.txt create mode 100644 evidence/native-sandbox/smoke.txt create mode 100644 evidence/native-sandbox/student-tests.txt create mode 100644 evidence/native-sandbox/summary.md create mode 100644 evidence/native-sandbox/telemetry-history.txt create mode 100644 evidence/native-sandbox/telemetry.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/race.txt create mode 100644 evidence/redis.txt create mode 100644 evidence/s3-security.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stack.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/summary.md create mode 100644 evidence/telemetry-history.txt create mode 100644 evidence/telemetry.txt create mode 100644 evidence/upload-memory.json create mode 100644 evidence/upload-memory.txt create mode 100644 faults.py create mode 100644 imaging.py create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/haproxy.cfg create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules/alerts.yml create mode 100644 infra/prometheus/rules/recording.yml create mode 100644 infra/prometheus/tests/alerts_test.yml create mode 100644 infra/rabbitmq.conf create mode 100644 infra/redis.conf create mode 100644 infra/tempo/tempo.yaml create mode 100644 janitor.py create mode 100644 main.py create mode 100644 migrate.py create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_text_as_bytes.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_jobs.sql create mode 100644 ops.py create mode 100644 pipeline.py create mode 100644 requirements.txt create mode 100644 s3_init.py create mode 100644 scripts/build_dashboard.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 scripts/worker.cmd create mode 100755 scripts/worker.sh create mode 100644 settings.py create mode 100644 storage.py create mode 100644 student-tests/check_configs.py create mode 100644 student-tests/check_dashboard.py create mode 100644 student-tests/check_stack.py create mode 100644 student-tests/collect_evidence.py create mode 100644 student-tests/common.py create mode 100644 student-tests/demo_instances.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/explain.sql create mode 100644 student-tests/failover_metrics.py create mode 100644 student-tests/fixtures/progressive.jpg create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/images.py create mode 100644 student-tests/incidents.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/obs.py create mode 100644 student-tests/redis_info.py create mode 100644 student-tests/rolling_update.py create mode 100644 student-tests/run_failover.py create mode 100644 student-tests/run_persistence.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_consistency.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_failures.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_janitor_replay.py create mode 100644 student-tests/test_jobs.py create mode 100644 student-tests/test_race.py create mode 100644 student-tests/test_s3_security.py create mode 100644 student-tests/test_telemetry.py create mode 100644 student-tests/upload_memory.py create mode 100644 student-tests/worker_probe.py create mode 100644 telemetry.py create mode 100644 uploads.py create mode 100644 worker.py diff --git a/.env.example b/.env.example index 2b32655..a57cdca 100644 --- a/.env.example +++ b/.env.example @@ -6,25 +6,65 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2): +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# SeaweedFS S3 (since lab4) +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url + +# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox); +# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher, +# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor. +BROKER_URL_FILE= WORKER_CONCURRENCY=2 +WORKER_PREFETCH=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=15 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=60 +WORKER_MEMORY_LIMIT_MB=512 STAGING_RETENTION_SECONDS=86400 +PIPELINE_VERSION=1 +# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the +# training stand by default; set FAULT_INJECTION=0 here to disable. + +# Pools and timeouts (same as lab3) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 + +# lab4: S3 client and uploads +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=5 +S3_MAX_ATTEMPTS=2 +S3_POOL_MAX=20 +UPLOAD_TIMEOUT_SECONDS=60 +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +DECODE_CONCURRENCY=2 +# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval +ATTACHMENT_GRACE_SECONDS=3600 +JANITOR_INTERVAL_SECONDS=300 + +# lab6: OpenTelemetry. Standard SDK variables; compose.yaml sets the same values for every app service +# and a distinct OTEL_SERVICE_NAME (messenger-api / messenger-worker / messenger-dispatcher / messenger-janitor). OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# true -> SDK off in all app processes (logs still go to stderr as JSON) +OTEL_SDK_DISABLED=false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c6d755c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf +*.yaml text eol=lf +*.yml text eol=lf +*.json text eol=lf +*.py text eol=lf +*.md text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..623eea0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бабиченко Максим, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесено всё решение lab5 (API, outbox, dispatcher, worker, janitor, тесты lab1–5). Добавлены: OpenTelemetry SDK во всех процессах (traces/metrics/logs по OTLP), W3C trace context API → outbox → RabbitMQ → worker (parent-child, replay — link), OTel Collector (memory_limiter, batch, ограниченные очереди), Prometheus (OTLP receiver, recording/alert rules + unit-тесты), Loki, Tempo, Grafana с provisioning источников и dashboard из Git, SLI/SLO, 6 алертов с runbooks, сценарии инцидентов. +Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`python scripts/init_secrets.py`, `docker compose up -d --build --wait`, `python student-tests/collect_evidence.py ...`). +Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев», `evidence/summary.md`, `evidence/incident-*.json`; прогон без Docker — `evidence/native-sandbox/`. +Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги» (одна копия каждого хранилища, буфер Collector в памяти, без TLS до lab7). +Источники и использование ИИ: REPORT.md, шапка (документация OTel/Prometheus/Loki/Tempo/Grafana; ИИ-ассистент Claude). -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [ ] make check и make test выполнены; приложен реальный вывод (Docker-прогон: `evidence/`). +- [ ] Сценарии отказов/безопасности своего уровня воспроизведены (Docker-прогон: `evidence/incident-*.json`). +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dc45121 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \ + pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py telemetry.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && mkdir -p /faults && chown appuser:appuser /faults +# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser +# копируется в пустой named volume при первом монтировании. + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index b97ae42..414451f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,368 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab5 (FastAPI, psycopg 3, pika, boto3, Pillow, RabbitMQ 4.1, SeaweedFS 3.80, PostgreSQL 17, Redis 7.4, HAProxy 3.2). Новое: OpenTelemetry Python SDK 1.45.2 (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) и `opentelemetry-instrumentation-psycopg` 0.66b2; образы `otel/opentelemetry-collector-contrib:0.164.0`, `prom/prometheus:v3.15.1`, `grafana/loki:3.7.9`, `grafana/tempo:3.1.1`, `grafana/grafana:12.4.14`, `busybox:1.37.0` (bootstrap прав volumes). Документация: OTel Collector configuration (memory_limiter, batch, exporterhelper sending_queue/retry_on_failure, internal telemetry), OTel semantic conventions (HTTP, database, messaging, `otel.sdk.*`), W3C Trace Context, Prometheus OTLP receiver и histograms/`histogram_quantile`, promtool unit testing, Loki OTLP ingestion и structured metadata, Tempo 3 single binary, Grafana provisioning (datasources, dashboards, derived fields, traces-to-logs). При написании кода, конфигураций и тестов использовался ИИ-ассистент (Claude); весь код я прогнал и могу объяснить и изменить на защите. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверяю на Windows (PowerShell, Docker Desktop). Python на хосте нужен только для тестов (стандартная библиотека). Перед запуском остановить стенд lab5 (`docker compose down` в его папке): заняты порты 8080/8404. Имя проекта Compose — `messenger-lab6`, volumes отдельные от lab5. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab6.git +cd mtusi2026lab6 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" +Copy-Item .env.example .env +python scripts/init_secrets.py # как в lab5 + secrets/grafana_admin_password (папка secrets/ в .gitignore) +docker compose --env-file .env config --quiet +docker compose up -d --build --wait # postgres, s3, rabbitmq, otel-collector, prometheus, loki, tempo -> + # migrate, s3-init, telemetry-init -> api x2, janitor, dispatcher, worker, + # grafana -> lb +docker compose ps -a # migrate, s3-init, telemetry-init: Exited (0); остальные running/healthy +python tests/smoke.py --base-url http://127.0.0.1:8080 # make test при включённой телеметрии + +# доказательства lab6 (~35 мин) в evidence/ и таблица замеров в этот отчёт: +python student-tests/collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history ` + incident-worker incident-db incident-telemetry incident-collector load +``` + +| Что | Где | +| --- | --- | +| Grafana (dashboard «Messenger — lab6 observability», Explore, Alerting) | http://127.0.0.1:3000, логин `admin`, пароль — `secrets/grafana_admin_password` | +| Prometheus: алерты pending/firing, запросы | http://127.0.0.1:9090/alerts, http://127.0.0.1:9090/graph | +| Loki / Tempo HTTP API (для тестов) | http://127.0.0.1:3100, http://127.0.0.1:3200 | +| Трасса конкретного ответа API | заголовок ответа `X-Trace-Id` (или свой `traceparent` в запросе) → Grafana → Explore → Tempo | + +Отдельные проверки (make на Windows нет): + +| Команда | Что проверяет | +| --- | --- | +| `python scripts/check_contract.py` | make check | +| `python student-tests/check_configs.py` | конфиги Collector/Prometheus/Loki/Tempo их же инструментами, unit-тесты алертов (promtool), provisioning Grafana через её API | +| `python student-tests/test_telemetry.py [request upload-chain retry-chain replay-link history]` | запрос ↔ лог ↔ трасса, цепочка upload → worker, retry, replay с link, сохранность истории | +| `python student-tests/check_dashboard.py` | каждый запрос dashboard и правил выполняется на живых данных, запрещённых меток нет | +| `python student-tests/incidents.py [worker-stop db-slow telemetry-down collector-down]` | инциденты: firing → диагностика → resolved | +| `docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml` | unit-тесты порогов/окон/`for` алертов | +| `python scripts/build_dashboard.py` | пересобрать `infra/grafana/dashboards/messenger.json` из описания панелей | + +Bootstrap: как в lab5 (`migrate`, `s3-init`, definitions RabbitMQ) плюс `telemetry-init` (одноразовый `chown 10001` volumes Loki и Tempo: оба процесса работают не от root). Provisioning Grafana — файлы `infra/grafana/provisioning` и `infra/grafana/dashboards`, монтируются только для чтения: источники данных и dashboard появляются при старте, ручного набора нет. Всё идемпотентно: повторный `up` ничего не пересоздаёт и не дублирует. + +Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults`, `promdata`, `lokidata`, `tempodata`, `grafanadata` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py`. Выключить телеметрию целиком (для сравнения): `OTEL_SDK_DISABLED=true` в `.env` и `docker compose up -d`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text + +--------------------------- сеть telemetry (internal) ----------------------------+ +Клиент -> lb -> api x2 ---|-- OTLP/HTTP :4318 --> otel-collector --+-- traces OTLP gRPC --> tempo (tempodata) | + | | (traces, metrics, memory_limiter |-- logs OTLP HTTP --> loki (lokidata) | + | | logs) batch, queues +-- metrics OTLP HTTP --> prometheus(promdata)| + PostgreSQL <-+ outbox | ^ pull :8888 (метрики Collector)| + (источник истины) | dispatcher, worker, janitor -- OTLP --^ | | + | grafana -------+--> prometheus, loki, tempo | + +----------------------------------------------------------------------------------+ + W3C traceparent: API (спан create, outbox) -> dispatcher (спан publish, AMQP headers + поле сообщения) + -> worker (спан process, consumer) -> S3/PostgreSQL; retry/replay -> новая строка outbox + 127.0.0.1: 8080 (lb), 8404 (stats), 3000 (grafana), 9090 (prometheus), 3100 (loki), 3200 (tempo) +``` + +Перенесено из lab5 без изменения бизнес-логики и контракта: всё приложение (API, outbox, dispatcher, worker, janitor, варианты изображений, повторы, DLQ, replay), миграции, тесты lab1–5, hooks. Бизнес-API не менялся; единственное расширение — необязательный заголовок ответа `X-Trace-Id` (trace_id запроса) и приём входящего `traceparent` (обратно совместимо, клиент может не знать о них). + +Что добавлено в lab6: +- `telemetry.py` — настройка OpenTelemetry SDK для всех процессов: ресурс, три провайдера (traces/metrics/logs) с OTLP/HTTP exporter, ограниченные очереди, сэмплер, JSON-логи в stderr + OTLP, метрики процесса/контейнера, W3C propagator, помощники спанов. +- Инструментирование: HTTP server-спаны и метрики в middleware `main.py`; спаны PostgreSQL (`opentelemetry-instrumentation-psycopg`, текст SQL с плейсхолдерами, без значений), S3 (`storage.py`), Redis и Argon2id (`auth.py`); messaging-спаны и перенос контекста через outbox/RabbitMQ (`pipeline.py`, `dispatcher.py`, `broker.py`, `worker.py`); метрики очереди/outbox (dispatcher), пула БД (`db.py`), занятости worker, времени до ready; структурные события логов (`event=...`). +- Конфигурации в Git: `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml`, `infra/prometheus/rules/{recording,alerts}.yml`, `infra/prometheus/tests/alerts_test.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, `infra/grafana/provisioning/**`, `infra/grafana/dashboards/messenger.json` (генерирует `scripts/build_dashboard.py`), `docs/runbooks.md`. +- `compose.yaml`: сервисы `otel-collector`, `telemetry-init`, `prometheus`, `loki`, `tempo`, `grafana`; сети `telemetry` (internal) и `ops` (только чтобы опубликовать UI на 127.0.0.1); volumes `promdata`, `lokidata`, `tempodata`, `grafanadata`; секрет `grafana_admin_password`. Restart hook перезапускает и хранилища телеметрии. +- Тесты: `student-tests/{obs.py, test_telemetry.py, check_dashboard.py, check_configs.py, incidents.py}`, обновлены `check_stack.py`, `run_persistence.py`, `collect_evidence.py`. + +### Роли и назначение хранилищ + +| Роль | Сервис | Версия | Зачем | Health | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Collector | `otel-collector` | otelcol-contrib 0.164.0 | приём OTLP, защита памяти, пакетирование, буфер на время недоступности хранилища, маршрутизация в 3 хранилища. **Не хранит** | метрики `otelcol_*` (scrape) — образ distroless, без shell | нет (очередь в памяти) | telemetry; 4317/4318/8888 не публикуются | нет | +| Метрики | `prometheus` | 3.15.1 | числовые ряды: «что, сколько, когда», percentiles из histogram, SLI, alert rules, состояние алертов | `/-/ready` | `promdata` (7 дней, ≤ 1 GB) | telemetry, ops; 127.0.0.1:9090 | нет | +| Логи | `loki` | 3.7.9 | события с контекстом: «почему» (error_type, request_id, trace_id) | — (`/ready` проверяют тесты) | `lokidata` (7 дней) | telemetry, ops; 127.0.0.1:3100 | нет | +| Трассы | `tempo` | 3.1.1 | причинная цепочка запроса через процессы и время каждого шага: «где» | — (`/ready`) | `tempodata` (7 дней) | telemetry, ops; 127.0.0.1:3200 | нет | +| UI | `grafana` | 12.4.14 | запросы к трём хранилищам, dashboard, связки лог ↔ трасса ↔ метрика; сама сигналы не хранит | `/api/health` | `grafanadata` (сессии/настройки UI) | telemetry, ops; 127.0.0.1:3000 | `grafana_admin_password` | + +Почему так: Grafana сама не хранит сигналы, а каждому сигналу нужно своё хранилище с подходящей моделью — TSDB для рядов (Prometheus умеет OTLP напрямую, поэтому без remote-write и без exporter-а в Collector), индекс по меткам + сжатые строки для логов (Loki принимает OTLP и хранит атрибуты записи как structured metadata, поэтому `trace_id`/`request_id` не становятся метками индекса), блоки трасс по trace_id (Tempo 3 single binary без Kafka). Collector отделяет приложение от хранилищ: приложение знает один адрес, а смена/недоступность хранилища его не касается. + +### Сигналы: контракт → реальные имена + +Имена OTel переводит в Prometheus `translation_strategy: UnderscoreEscapingWithSuffixes` (зафиксировано в `prometheus.yml`): точки → `_`, суффикс единицы (`_seconds`, `_bytes`) и `_total` для счётчиков; `service.name` → метка `job`, `service.instance.id` → `instance`, остальные ресурсные атрибуты — в `target_info`. + +| Сигнал контракта (уровень) | Инструмент OTel (единица) | В хранилище | Метки / поля | +| --- | --- | --- | --- | +| HTTP requests (3) | `http.server.request.duration` histogram, его `_count` | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`, `http_response_status_class` (`2xx`…) | +| HTTP duration (3) | то же, секунды; buckets 5 мс … 10 с | `http_server_request_duration_seconds_bucket` | те же + `le` | +| Jobs processed / failed (3) | `messenger.job.duration` histogram, `_count` (одна метрика, без дубля) | `messenger_job_duration_seconds_count` | `job_type="attachment.process.v1"`, `outcome` = ready / retry / failed / stale / lease_lost / db_error | +| Job processing duration (3) | `messenger.job.duration` (s), buckets 0,1 … 120 с | `messenger_job_duration_seconds_bucket` | те же | +| Доставки/публикации (доп.) | `messenger.messages.consumed`, `messenger.messages.published` counters | `messenger_messages_consumed_total`, `..._published_total` | `messaging_destination_name`, `outcome` (processed/noop/dead_lettered/requeued; ok/error) | +| Логи API и worker (3) | Python `logging` → OTel Logs API | Loki: метки `service_name`, `service_instance_id`, `deployment_environment_name`; structured metadata `event`, `severity_text`, `trace_id`, `span_id`, `request_id`, `job_id`, `error_type`… | в stderr та же запись одной JSON-строкой (`ts`, `severity`, `service.name`, `event`, `trace_id`, `span_id`) | +| Трассы API и worker (3) | server / producer / consumer / client / internal спаны | Tempo | `http.route`, `http.response.status_code`, `messaging.*`, `messenger.job.*`, `db.statement`, `aws.s3.key`, `error.type` | +| Queue depth / oldest job age (4) | `messenger.queue.depth` gauge; `messenger.queue.oldest_age` (s); доп. `messenger.jobs.backlog`, `messenger.queue.consumers`, `messenger.outbox.pending`, `messenger.outbox.oldest_age` (s) | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds`, … | `messaging_destination_name` (очередь/DLQ), `status` | +| DB pool used/waiting (4) | `db.client.connection.count` (`state`=used/idle), `.max`, `.pending_requests`, `.timeouts`, `db.client.connection.wait_time` (s, histogram) | `db_client_connection_*` | `db_client_connection_pool_name="messenger-api"`, `db_client_connection_state` | +| Worker busy/capacity (4) | `messenger.worker.busy`, `messenger.worker.capacity` | `messenger_worker_busy`, `messenger_worker_capacity` | `messaging_destination_name` | +| Process / container CPU, RAM (4) | `process.cpu.time` (s), `process.memory.usage` (By); `container.cpu.time`, `container.cpu.limit`, `container.memory.usage`, `container.memory.limit` (cgroup v2, только в Docker) | `process_cpu_time_seconds_total`, `process_memory_usage_bytes`, `container_*` | `cpu_mode` | +| End-to-end time to ready (4) | `messenger.attachment.time_to_ready` (s), buckets 0,5 … 600 с; от COMMIT задания в API до COMMIT ready, по часам PostgreSQL | `messenger_attachment_time_to_ready_seconds_*` | `job_type` | +| Сквозная связь upload → publish → consume → S3/DB (4) | W3C `traceparent`/`tracestate`: parent-child, replay — link | Tempo | см. ниже | +| Потери телеметрии (4) | SDK: `otel.sdk.processor.span.processed{error.type}`, `otel.sdk.exporter.span.exported{error.type}`; Collector: `otelcol_exporter_{sent,send_failed,enqueue_failed}_*`, `otelcol_exporter_queue_size/capacity`, `otelcol_receiver_refused_*` | Prometheus | `exporter`, `error_type` | + +Ресурс (одинаков для всех трёх сигналов процесса): `service.name` = `messenger-api` / `messenger-worker` / `messenger-dispatcher` / `messenger-janitor` (и `messenger-ops` у `ops.py`) — задаётся в коде процесса, поэтому общий `.env` не делает janitor «api»; `service.version` = `APP_VERSION`; `service.instance.id` = hostname контейнера (тот же, что `X-Instance-Id`); `deployment.environment.name=lab` (`OTEL_RESOURCE_ATTRIBUTES`). + +**Что не попадает в телеметрию.** Метки метрик — только перечисленные выше (метод, шаблон маршрута, класс/код статуса, тип/исход задания, очередь, состояние соединения, режим CPU); `check_dashboard.py` проверяет, что у метрик приложения нет `user_id`, `chat_id`, `message_id`, `attachment_id`, `request_id`, URL, имени файла, текста и значений-UUID. В спанах и логах есть технические ID синтетических данных (`job_id`, `attachment_id`, `request_id`, ключ S3 вида `attachments//v1/crop.jpg`) — без них нельзя пройти от загрузки к результату worker. Не пишутся: пароли, `Authorization`, токен сессии и его хеш (спаны Redis без ключа), текст сообщения, имя файла, байты изображения, значения параметров SQL (`capture_parameters=False`), signed URL (их нет). `test_telemetry.py request` ищет текст сообщения, пароль и токен в Loki и атрибутах спанов — 0 совпадений. В Collector дополнительно `attributes/scrub` удаляет любой атрибут, чьё имя похоже на password/authorization/cookie/secret/token/api key. + +### Перенос контекста через очередь + +```text +[api] POST /api/v1/chats/{chat_id}/attachments SERVER + ├─ PostgreSQL INSERT attachments, S3 PutObject staging/, UPDATE ... state='queued' + └─ create attachments PRODUCER ← строка outbox в ТОЙ ЖЕ транзакции; traceparent этого спана + │ пишется в attachment_jobs.traceparent и в payload outbox + └─ [dispatcher] publish attachments PRODUCER ← родитель = traceparent из outbox; свой контекст + │ кладёт в AMQP header traceparent и в поле сообщения + └─ [worker] process attachments.process.v1 CONSUMER ← родитель = header (запасной — поле тела) + ├─ claim (PostgreSQL), S3 GetObject, imaging, S3 PutObject x4, commit result (PostgreSQL) + ├─ (ошибка) create attachments PRODUCER → publish → process (попытка 2) — та же трасса + └─ cleanup staging (S3 DeleteObject, UPDATE) — после ack, в той же трассе +``` + +Выбранная семантика: **parent-child** вдоль причинной цепочки (публикация и обработка — продолжение запроса пользователя, а задержка между ними видна как разрыв во времени). **Link** — когда это новое решение, а не продолжение: `ops.py replay` создаёт новую трассу (корень `ops replay`), спан `create` которой ссылается на исходную загрузку. Восстановление истёкшей аренды dispatcher-ом и сверка «queued без сообщения» создают спаны `recover expired lease` / `republish lost message` с родителем из `attachment_jobs.traceparent`. В lab5 `traceparent` был случайным; теперь это контекст реального спана. + +Сэмплирование: `ParentBased(AlwaysOn)`, но клиентский спан без родителя не записывается — это фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, проверка соединений пула). Всё, что относится к запросу или заданию, сохраняется полностью (100%). Проверки здоровья (`/health/*`, HAProxy раз в секунду) — только в метриках, без трасс. + +### SLI / SLO и окно + +| SLO | SLI (числитель / знаменатель) | Цель | Окно | Recording rule | +| --- | --- | --- | --- | --- | +| SLO-1 доступность API | бизнес-запросы API без 5xx / все бизнес-запросы API (все маршруты кроме `/health/*`). 4xx — ошибки клиента, не входят в числитель | ≥ 99% | скользящий 1 ч (учебное окно; в эксплуатации то же выражение с 28d) | `messenger:slo_availability:ratio_1h`, бюджет `messenger:slo_availability:error_budget_remaining_1h` | +| SLO-2 задержка записи | успешные `POST /api/v1/chats/{chat_id}/messages` быстрее 250 мс (bucket `le="0.25"`) / все успешные такие запросы | ≥ 95% | 1 ч | `messenger:slo_latency_messages:ratio_1h` | +| SLO-3 фоновая обработка | вложения ready не позже 30 с от приёма (bucket `le="30"` гистограммы time_to_ready) / все ставшие ready | ≥ 95% | 1 ч | `messenger:slo_time_to_ready:ratio_1h` | + +Проверяемость: все SLI — отношения счётчиков/buckets гистограмм, просуммированных по репликам (`sum(increase(...[1h]))`), поэтому SLO вычисляется одинаково на любом наборе реплик и видно на dashboard (строка «Обзор и SLO»). + +### Dashboard: запросы и единицы + +Dashboard — `infra/grafana/dashboards/messenger.json` (импортируется provisioning-ом, папка Messenger; 34 панели в 6 строках: обзор/SLO и алерты, API RED, фоновая обработка, насыщение, конвейер телеметрии, логи и трассы). Ключевые запросы (`$__rate_interval` подставляет Grafana; `API` = `job="messenger-api", http_route!~"/health/.*"`): + +| Панель | PromQL | Единица | +| --- | --- | --- | +| RPS по маршрутам / по классу статуса | `sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{API}[$__rate_interval]))` | req/s | +| Доля 5xx | `(sum(rate(..._count{API, http_response_status_class="5xx"}[…])) or on() vector(0)) / sum(rate(..._count{API}[…]))` | доля (0–1) | +| p50 / p95 / p99 API | `histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{API}[$__rate_interval])))` | с | +| p95 по маршрутам | `histogram_quantile(0.95, sum by (le, http_route) (rate(..._bucket{API}[…])))` | с | +| Попытки заданий по исходу | `sum by (outcome) (rate(messenger_job_duration_seconds_count[…]))` | 1/с | +| Длительность попытки / время до ready p50/p95/p99 | `histogram_quantile(q, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="ready"}[…])))`, то же для `messenger_attachment_time_to_ready_seconds_bucket` | с | +| Глубина очереди и backlog | `max by (messaging_destination_name) (messenger_queue_depth)`, `max by (status) (messenger_jobs_backlog)` | сообщений / заданий | +| Возраст старейшего задания и outbox | `max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})`, `max(messenger_outbox_oldest_age_seconds)` | с | +| Пул БД | `sum by (instance) (db_client_connection_count{db_client_connection_state="used"})`, `max(db_client_connection_max)`, `sum by (instance) (db_client_connection_pending_requests)`; p95 ожидания `histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[…])))` | соединений / с | +| Насыщение worker | `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` | доля | +| CPU / RAM | `sum by (job, instance) (rate(process_cpu_time_seconds_total[…]))` (ядер), `rate(container_cpu_time_seconds_total) / container_cpu_limit` (доля лимита), `process_memory_usage_bytes`, `container_memory_usage_bytes` | ядра / доля / байты | +| Collector и SDK | `rate(otelcol_receiver_accepted_spans[…])`, `rate(otelcol_exporter_send_failed_spans[…])`, `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`, `otelcol_process_memory_rss`, `rate(otel_sdk_exporter_span_exported_total{error_type!=""}[…])` | 1/с, доля, байты | +| Логи / трассы | Loki `{service_name=~"messenger-.*"} \| detected_level=~"warn\|error"`; Tempo TraceQL `{ status = error }` | — | + +Percentile только из buckets гистограммы, просуммированных по всем репликам **до** `histogram_quantile` (`sum by (le)`); среднее и усреднение p95 реплик не используются. Точность — ширина bucket: например, при запросах по 5,0 с (statement_timeout) интерполяция внутри bucket 5–7,5 с даёт p95 ≈ 7,4 с; это ограничение метода, видно на инциденте 2. + +### Алерты + +`infra/prometheus/rules/alerts.yml`; состояние — Prometheus `/alerts` и `ALERTS{}` (видно и resolved по истории), Grafana показывает их как data source rules. Внешней доставки нет (по заданию). + +| Алерт | Условие | Окно | Порог | for | Runbook | +| --- | --- | --- | --- | --- | --- | +| MessengerHighErrorRate | доля 5xx бизнес-запросов API и трафик > 0,2 запроса/с | rate 1m | > 5% (в 5 раз выше бюджета SLO-1) | 1m | `docs/runbooks.md#messengerhigherrorrate` | +| MessengerQueueLagGrowing | возраст старейшего готового задания | мгновенное значение (обновление 5–10 с) | > 30 с (SLO-3) | 1m | `#messengerqueuelaggrowing` | +| MessengerWorkerNoConsumers | consumers очереди = 0 | — | 0 | 1m | `#messengerworkernoconsumers` | +| MessengerDbPoolSaturated | запросы ждут соединение из пула | — | > 0 | 1m | `#messengerdbpoolsaturated` | +| TelemetryExportFailing | потери в Collector (send/enqueue failed) **или** очередь exporter заполнена > 5% | rate 2m | > 0 / > 5% | 1m | `#telemetryexportfailing` | +| TelemetryCollectorDown | `up{job="otel-collector"} == 0` | — | — | 1m | `#telemetrycollectordown` | + +Пороги, окна и `for` проверены unit-тестами `infra/prometheus/tests/alerts_test.yml` (promtool: не срабатывает раньше `for`, не срабатывает при пустом трафике и на ошибках `/health`, resolved после восстановления) и живыми инцидентами ниже. + +### Collector и SDK: память и недоступность хранилища + +- SDK (каждый процесс): `BatchSpanProcessor` и `BatchLogRecordProcessor` с очередью 2048 (`OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_QUEUE_SIZE`), экспорт в фоновом потоке, таймаут 5 с (`OTEL_EXPORTER_OTLP_TIMEOUT`); метрики — агрегаты фиксированного размера раз в 10 с. Переполнение и непринятые пакеты считаются самим SDK (`OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED=true`). Ошибки экспорта пишутся в лог не чаще раза в 30 с и не отправляются тем же экспортом (иначе петля). +- Collector: первым в каждом pipeline `memory_limiter` (heap Go: жёсткий предел 200 MiB, мягкий 160 MiB — receiver отвечает клиентам ошибкой, SDK отбрасывает у себя), затем `batch` (2 с / 1024–2048 элементов). У каждого exporter своя `sending_queue` в памяти (Tempo, Loki — 200 пакетов, Prometheus — 100) и `retry_on_failure` (1 → 10 с, `max_elapsed_time` 120 с). Окно повторов считается для пакета с первой попытки; ожидание в очереди в него не входит, поэтому буфер на время недоступности — это очередь. Пакет отбрасывается, когда истекли его 120 с повторов (`otelcol_exporter_send_failed_*`) или очередь полна (`otelcol_exporter_enqueue_failed_*`). Pipeline независимы: недоступный Tempo не мешает логам и метрикам. Очередь не персистентная: рестарт Collector теряет буфер (осознанный компромисс учебного стенда, `file_storage` не нужен). +- Контейнер Collector 384 MiB: RSS = heap (≤ 200 MiB) + runtime Go + отображённый в память бинарник contrib (~150 MiB вытесняемых страниц); поэтому тест сравнивает с лимитом именно heap (`otelcol_process_runtime_heap_alloc_bytes`), а RSS/память контейнера приводит для сведения. + +### Соответствие `.env.example` + +| Переменная | Что делает в этом решении | +| --- | --- | +| всё из lab5 | как в lab5 (см. REPORT lab5) | +| OTEL_SERVICE_NAME | `messenger-api` для api; compose задаёт свой каждому сервису; фактически `service.name` берётся из кода процесса | +| OTEL_EXPORTER_OTLP_ENDPOINT / _PROTOCOL | `http://otel-collector:4318`, `http/protobuf` (SDK добавляет `/v1/traces`, `/v1/metrics`, `/v1/logs`) | +| OTEL_RESOURCE_ATTRIBUTES | `deployment.environment.name=lab` | +| OTEL_SDK_DISABLED | `true` — SDK выключен во всех процессах (логи остаются JSON в stderr); для сравнения telemetry off/on | +| (только compose) OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_METRIC_EXPORT_INTERVAL, OTEL_BSP_MAX_QUEUE_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED | 5 с, 10000 мс, 2048, 2048, `true` | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +`python student-tests/collect_evidence.py …` пишет вывод каждого шага в `evidence/<шаг>.txt` (дата, commit, код выхода), итог — `evidence/summary.md`, JSON инцидентов — `evidence/incident-*.json`, а таблицы замеров — в блок «Замеры (Docker)» ниже. Те же сценарии прогнаны в нативной песочнице (те же версии Collector/Prometheus/Loki/Tempo, процессы без Docker; см. «Где проверено») — `evidence/native-sandbox/`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker-стенд, `evidence/`; то же в нативной песочнице — `evidence/native-sandbox/`) | Файл доказательства | +| --- | --- | --- | --- | +| 3: минимум lab5 работает, `make test` при включённой телеметрии | `tests/smoke.py` | PASS через балансировщик с SDK во всех процессах | smoke.txt | +| 3: Collector, 3 хранилища и Grafana в одном воспроизводимом стенде; конфиги/provisioning в Git; persistent volumes | `check_stack.py`, `check_configs.py`, `test_telemetry.py history` | конфиги валидны инструментами тех же версий; unit-тесты алертов SUCCESS; после restart prometheus/loki/tempo прежние трасса, логи и точка метрики находятся | stack.txt, configs.txt, telemetry-history.txt | +| 3: API и worker отдают OTLP; RPS/errors/duration API, jobs ok/failed и duration worker; логи и реальные трассы | `check_dashboard.py`, `test_telemetry.py` | все 54 PromQL-выражения dashboard и 16 выражений правил выполняются без ошибок, ключевые сигналы с данными; трассы API, dispatcher, worker | dashboard.txt, telemetry.txt | +| 3: импортируемый dashboard; низкая кардинальность; trace/span id в логах; нет содержимого/секретов | `check_dashboard.py`, `check_configs.py` (Grafana API), `test_telemetry.py request` | метки только из белого списка, `http_route` — шаблоны; dashboard provisioned=true; текст, пароль, токен: 0 совпадений в Loki и спанах | dashboard.txt, configs.txt, telemetry.txt | +| 3: контролируемый запрос → trace + лог; job → worker trace + лог | `test_telemetry.py request upload-chain` | свой `traceparent` → server-спан с дочерними PostgreSQL/Redis → лог `event=http.request` с тем же trace_id/span_id и request_id = `X-Request-Id`; логи worker `job.claimed`/`job.ready` с trace_id загрузки | telemetry.txt | +| 3: назначение каждого хранилища | раздел «Роли и назначение хранилищ» | — | REPORT.md | +| 4: W3C через API → broker → worker, producer/consumer, цепочка до S3/DB, включая retry | `test_telemetry.py upload-chain retry-chain replay-link` | одна трасса: server → create (producer) → publish (producer, dispatcher) → process (consumer, worker) → 4×S3 PutObject + commit; retry: попытка 1 ERROR `storage_unavailable` → create → publish → попытка 2 ready; replay — новая трасса с link на исходную | telemetry.txt | +| 4: p50/p95/p99 из histogram, доля ошибок, queue depth/oldest age, DB pool, saturation worker; запросы и единицы | dashboard + раздел «Dashboard: запросы и единицы» | все панели выполняются на живых данных | dashboard.txt | +| 4: SLI/SLO и окно; alert rules high error rate и queue lag; проверены отказом (firing/resolved) | `incidents.py worker-stop db-slow`, `promtool test rules` | см. таблицы инцидентов: firing и resolved по каждому алерту | incident-worker.txt, incident-db.txt, configs.txt | +| 4: memory limiter/batch, ограниченная буферизация; остановка хранилища телеметрии не ломает сообщения и не растит RAM | `incidents.py telemetry-down collector-down` | API 100% успешных ответов, heap Collector ограничен, потери посчитаны | incident-telemetry.txt, incident-collector.txt | +| 4: разбор двух инцидентов и шаги диагностики | раздел «Эксперименты и отказы» | — | REPORT.md, docs/runbooks.md | +| Выбранный трек 5 | — | не выбран | — | + +### Где проверено + +**Нативная песочница (выполнено).** Среда, где писалось решение, не имеет доступа к Docker Hub, поэтому весь стенд был поднят теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API за балансировщиком, dispatcher, worker, janitor (Python 3.11, те же `requirements.txt`) и **те же версии** otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из `infra/` (заменены только адреса `tempo:4317` → `127.0.0.1:14317` и т.п.). Тесты запускались без изменений через тонкую подмену команды `docker compose` (stop/start/exec на процессы). Результаты — `evidence/native-sandbox/`, цифры — в разделе ниже. Grafana в песочнице не запускалась (нет бинарника без Docker Hub): её provisioning проверяет `check_configs.py` через HTTP API Grafana на Docker-стенде, а все запросы dashboard выполнены против живых Prometheus/Loki/Tempo (`check_dashboard.py`). + +**Docker-стенд (Windows 10, Docker Desktop 28.1.1, Compose 2.35.1; VM 16 CPU / 15,2 GiB) — выполнено 2026-11-01.** Стенд поднят из чистого состояния (`init_secrets.py`, `docker compose pull`, `up -d --build --wait`) без правок кода и конфигов: `migrate`, `s3-init`, `telemetry-init` — Exited (0), остальные running/healthy; Loki и Tempo пишут в volumes от uid 10001 без ошибок прав. Полный прогон `collect_evidence.py` — все шаги PASS: lab6 (check, stack, smoke, configs, telemetry, dashboard, telemetry-history, 4 инцидента, load) и регрессия lab1–5 (`--from jobs-variants`, 22 шага, включая failover и persistence; restart hook вместе с ожиданием готовности — 60,9 с при лимите 90 с). Вывод — `evidence/*.txt`, сводка — `evidence/summary.md`, таблицы — блок «Замеры (Docker)» ниже. Grafana проверена через её API: 3 источника данных healthy, dashboard provisioned, трасса по `X-Trace-Id` открывается через источник Tempo, лог с тем же `trace_id` — через Loki (связки tracesToLogs и derived field TraceID). ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. По сохранённым метрикам, логам и трассам можно найти причину инцидента, не заходя в контейнеры: остановленный worker отличается от медленной БД, а медленная БД — от медленного обработчика. +2. Недоступность любого звена телеметрии (хранилище, Collector) не влияет на ответы API, память Collector и процессов приложения ограничена, а потери телеметрии считаются. + +Методика: `student-tests/incidents.py` — нагрузка (closed loop, POST сообщений), контролируемый отказ, опрос `/api/v1/alerts` каждые 5 с до firing, «диагностика» только запросами к Prometheus/Loki/Tempo, устранение отказа, ожидание resolved; хронология в выводе, итог в `evidence/incident-*.json` (без токенов и текстов). + +### Нативная песочница: результаты + +Окружение: Linux x86_64, 2 vCPU, 8 GiB RAM на всё (стенд, хранилища телеметрии и клиент нагрузки); PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, Python 3.11; otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 (те же версии и конфиги, что в compose.yaml). Дата прогона 2026-11-01. Сводка шагов (все PASS, включая регрессию lab1–5: варианты изображений, outbox, повторы/DLQ/replay, конкуренция worker, вложения, идемпотентность, гонки) — `evidence/native-sandbox/summary.md`. + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с | +| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) | + +Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной). + +### Инцидент 1: остановлен worker («задания не обрабатываются») + +Воздействие: `docker compose stop worker`, затем 3 загрузки вложений (API отвечает 202 queued — приём от worker не зависит). + +Шаги диагностики (как в `docs/runbooks.md`): +1. **Алерт** `MessengerWorkerNoConsumers` (consumers = 0), затем `MessengerQueueLagGrowing` (возраст старейшего задания > 30 с). Dashboard «Возраст старейшего задания» растёт линейно — задания не забирают, а не «их стало много». +2. **Публикация или обработка?** `messenger_outbox_oldest_age_seconds` = 0 и `messenger_messages_published_total{outcome="ok"}` растёт → dispatcher и RabbitMQ работают, сообщения в брокере (`messenger_queue_depth` = числу загрузок). +3. **Кто не забирает?** `messenger_queue_consumers` = 0, метрики `messenger_worker_*` перестали обновляться (worker не экспортирует), последний лог worker в Loki — `event=service.stopped` (штатная остановка, а не падение — тогда был бы `job.handler_crashed`/обрыв без stopped). +4. **Трасса любой из загрузок** (trace_id из лога API `event=http.request` или `X-Trace-Id`): есть `publish attachments`, нет `process attachments.process.v1` — сообщение доставлено в очередь, но не взято. +5. Причина: worker остановлен. Действие: `docker compose start worker`. Проверка: вложения ready через 2–5 с (Docker 4,7 с, песочница 2,0 с), backlog → 0, оба алерта resolved (Docker — через 29,7 с после start). + +### Инцидент 2: медленная БД (блокировка таблицы `messages`) vs медленный обработчик + +Воздействие: под нагрузкой 8 потоков `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` в отдельной транзакции на 210 с (PostgreSQL жив, `/health/ready` 200, но каждая запись ждёт блокировку и через `statement_timeout` 5 с отменяется → API 503). + +Шаги диагностики: +1. **Алерт** `MessengerHighErrorRate`. Срабатывает не сразу: окно rate 1m заполнено быстрыми запросами до инцидента (~115–130/с), а во время блокировки closed-loop нагрузка даёт ~1,6 запроса/с (каждый висит 5 с), поэтому доля > 5% появляется только через 50–80 с, плюс `for: 1m` (firing через 140,8 с от LOCK в Docker, 120,1 с в песочнице). +2. **Какие маршруты и коды**: только `POST /api/v1/chats/{chat_id}/messages`, код 503; чтения других таблиц (`GET /api/v1/users/me`, `GET /api/v1/chats`) во время блокировки 5–20 мс (Docker 17,2 мс, песочница 4,7 мс) → не «всё медленно», а конкретная таблица/операция. +3. **БД или обработчик — по трассе**: TraceQL `{ resource.service.name = "messenger-api" && status = error }` → server-спан ≈ 5005–5015 мс, из них дочерний `INSERT` (PostgreSQL) ≈ 5001 мс со статусом ERROR — 99,8–99,9% времени запрос ждал БД. Для контраста медленный **обработчик**: `POST /api/v1/users` — server 20–35 мс, из них `argon2id hash` 16–29 мс (CPU в процессе API), все спаны БД < 1 мс. По метрикам то же различие: при медленной БД растут `db_client_connection_count{state="used"}` (соединения заняты ожиданием) и p95 конкретного маршрута, при медленном обработчике пул свободен, а растёт CPU процесса API (`process_cpu_time_seconds_total`). +4. **Почему — по логам**: `{service_name="messenger-api"} | event="db.unavailable"` с `error_type=QueryCanceled` (statement_timeout), а не `OperationalError` (БД недоступна) или `PoolTimeout` (нет соединений). +5. **Подтверждение в PostgreSQL**: `pg_stat_activity` показывает сессии с `wait_event_type=Lock` и держателя блокировки (`LOCK TABLE ...`). Действие: завершить транзакцию-держателя. Проверка: 5xx → 0, алерт resolved через 30–50 с после снятия блокировки (Docker 50,1 с, песочница 30,0 с: окно rate 1m заполняется успешными запросами, затем ближайшая оценка правила). + +### Инцидент 3: недоступна телеметрия (Tempo + Loki; затем сам Collector) + +Ожидаем: API не замечает отказа; Collector копит пакеты в ограниченной очереди, heap не выходит за `memory_limiter`; `TelemetryExportFailing` firing → resolved; после восстановления очередь досылается, потери посчитаны. Без Collector: SDK каждого процесса отбрасывает лишнее (очередь 2048), память API не растёт, потери видны в `otel_sdk_exporter_span_exported_total{error_type!=""}` после восстановления. + +Что показал эксперимент (числа — в таблицах выше): +- Tempo + Loki остановлены на 150 с под нагрузкой ~60 записей/с (Docker; в песочнице ~68/с): все ответы API 201 (9555 запросов), p95 не изменился (31 мс до и во время отказа). Очереди exporter Tempo/Loki росли (метрики продолжали идти в Prometheus — pipeline независимы), heap Collector вырос с 35,5 до max 86,9 MiB (песочница 44 → 93 MiB) — далеко от предела 200 MiB, память контейнера Collector не выше 107 MiB; `TelemetryExportFailing` перешёл в firing через 73 с (песочница ~100 с) — по заполнению очереди, ещё до первых потерь. После start очереди досланы за 60 с (песочница ~80 с), алерт resolved через 166 с после start (песочница ~155 с: окно rate 2m по счётчику потерь — последние пакеты с истёкшим окном повторов отброшены около момента восстановления, плюс `for`); все 13 пробных трасс, отправленных во время отказа, нашлись в Tempo (песочница 14 из 15). Потеряно и посчитано Collector-ом 4096 спанов и 235 записей логов (песочница 4096 и 269) — пакеты, у которых истекло окно повторов. +- Collector остановлен на 90 с: все ответы API 201 (5852 запроса), память двух реплик API 295 → max 314 MiB (+18 MiB; песочница +12 MiB) и после восстановления не растёт. Экспорт в SDK получает ошибку соединения, пакет отбрасывается (буфера на диске нет). В Docker остановленный контейнер исчезает из DNS сети, и каждая попытка экспорта висит до таймаута 5 с. Экспорт становится медленнее, чем API создаёт спаны, поэтому ограниченная очередь процессора (2048) переполняется и лишнее отбрасывается сразу: за отказ потеряно и посчитано 8252 спана `processor_queue_full` и 10752 `exporter_failed` (`otel_sdk_processor_span_processed_total`, `otel_sdk_exporter_span_exported_total` с `error_type`). В песочнице соединение отклонялось мгновенно: 24,6 тыс. `exporter_failed`, очередь не переполнялась. В обоих случаях память ограничена очередью и ответы API не замедлились (p95 31–32 мс); после start экспорт возобновился без перезапуска приложения (10287 спанов за минуту). +- Вывод: отказ любого звена телеметрии не затрагивает пользователей и не растит память без предела; потери ограничены временем отказа и видны в метриках. Короткую недоступность хранилища (до заполнения очереди Collector, здесь минуты) Collector переживает почти без потерь, недоступность самого Collector — нет. + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB. + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с | +| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) | + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 | +| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 | +| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 | +| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 | +| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | + ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается: недоступность любого хранилища телеметрии и самого Collector — API и обработка вложений продолжают работать; память ограничена очередями SDK (2048 на сигнал и процесс) и Collector (memory_limiter, очереди фиксированного размера). Переживают рестарт хранилищ (volumes): метрики, логи, трассы, настройки Grafana. +- Теряется (и считается): телеметрия, отправленная, пока Collector недоступен (SDK не буферизует на диск); пакеты, у которых истекло окно повторов Collector или очередь переполнена; буфер Collector при его рестарте. Метрики приложения за время недоступности Collector — пропуск на графиках (cumulative-счётчики после восстановления продолжают с верного итога). +- Один экземпляр каждого хранилища и Collector на одном Docker-host — нет HA телеметрии; при отказе Prometheus алерты не вычисляются (нужен внешний «мета-мониторинг» — вне учебного стенда). +- Метрики очереди считает один dispatcher: если он упал, значения `messenger_queue_*` устаревают (видно по отсутствию свежих точек); алерт `absent()` на них — следующий шаг. +- Учебные упрощения: внутренние соединения OTLP и UI без TLS (lab7), Prometheus/Loki/Tempo опубликованы на 127.0.0.1 для тестов и защиты; сэмплирование трасс 100% (для учебной нагрузки; стоимость и баланс сэмплирования — трек 5.1, не заявлен); burn-rate алерты с несколькими окнами — трек 5.2, не заявлен. +- Подтверждено измерением: алерты срабатывают и снимаются на контролируемых отказах; диагностика отличает остановленный worker, медленную БД и медленный обработчик; отказ телеметрии не влияет на API. Гипотеза без измерения: накладные расходы телеметрии на задержку и CPU (сравнение `OTEL_SDK_DISABLED=true/false` при одинаковой нагрузке) — трек 5.1. + +## Ответы на вопросы защиты (кратко) + +- **Лог, метрика и trace об одном инциденте.** Метрика отвечает «что и когда, насколько массово» (доля 5xx выросла в 14:02, только на POST messages) — дёшево, агрегатно, для алертов. Трасса — «где именно» в одном запросе (5 с из 5,004 с — `INSERT` в PostgreSQL), через все процессы. Лог — «почему» с деталями события (`QueryCanceled`, request_id, job_id). Связаны через `trace_id`: из алерта → error-трасса → её логи. +- **Почему `chat_id` в labels может исчерпать хранилище.** Каждая уникальная комбинация меток — отдельный временной ряд со своим индексом и чанками в памяти. Чатов — неограниченное и растущее число, умноженное на маршруты, коды и buckets гистограммы → миллионы рядов, рост RAM/диска TSDB и медленные запросы. Поэтому в метрики — только ограниченные множества, а ID — в трассы/логи. +- **Как очередь переносит причинную связь между процессами.** Producer записывает W3C `traceparent` (trace_id + id своего спана) в сообщение — здесь в строку outbox, затем dispatcher в AMQP header и поле тела; consumer извлекает его и начинает свой спан с этим родителем (или с link). Так trace_id один для HTTP-запроса и фоновой обработки, хотя между ними секунды и разные процессы. +- **Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик.** Среднее прячет хвост: 99 запросов по 10 мс и один по 5 с дают среднее 60 мс, а пользователь видит 5 с. Перцентиль — не аддитивная величина: p99 двух реплик с разной нагрузкой нельзя усреднить (реплика с 10 запросами весила бы как реплика с 10 000). Правильно — сложить buckets гистограмм всех реплик (`sum by (le)`) и уже из суммы вычислить `histogram_quantile`. diff --git a/attachments.py b/attachments.py new file mode 100644 index 0000000..dabbde2 --- /dev/null +++ b/attachments.py @@ -0,0 +1,80 @@ +"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202. + +Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого +объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение: + + 1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503. + 2. PUT исходника в staging/ — упало: удаляем объект (PUT мог пройти) + и строку, клиенту 503. Если и это не вышло — + uploading-строку уберёт janitor после grace. + 3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT + — упало/неизвестно: клиенту 503, строка + остаётся uploading -> janitor удалит объект + и строку. Если COMMIT всё же прошёл, задание + есть и будет обработано (повтор клиента даст + второе вложение — в контракте нет ключа). + 4. 202 Attachment(status=queued) — только после шага 3. + +Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный +брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно. +""" +import logging +from uuid import UUID, uuid4 + +import db +import settings +import storage +from errors import unavailable +from uploads import ImageInfo, Upload + +log = logging.getLogger("messenger.attachments") + + +def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict: + attachment_id = uuid4() + key = storage.new_key(uuid4().hex) + db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type, + upload.size, upload.sha256, image.width, image.height, key) + try: + storage.put(key, upload.file, upload.size, image.content_type, upload.sha256) + except storage.StorageUnavailable: + compensate(attachment_id, key) + raise unavailable("File storage is temporarily unavailable") from None + row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID) + if row is None: + # janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace). + log.warning("Upload %s was reclaimed before it was queued", attachment_id) + raise unavailable() + log.info("Attachment %s accepted into staging and queued", attachment_id) + return {**row, "variants": []} + + +def compensate(attachment_id: UUID, key: str) -> None: + """Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor.""" + try: + storage.delete(key, quick=True) + db.delete_upload_intent(attachment_id) + log.info("Upload %s rolled back after S3 failure", attachment_id) + except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту + log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__) + + +def public(row: dict) -> dict: + """Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей).""" + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["state"], + "variants": [ + {key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for v in row.get("variants", []) + ], + "error_code": row["error_code"] if row["state"] == "failed" else None, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..5f2a343 --- /dev/null +++ b/auth.py @@ -0,0 +1,150 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +import telemetry +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + # Argon2id нагружает CPU обработчика (а не БД): в трассе это отдельный спан. + with telemetry.internal_span("argon2id hash"): + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + with telemetry.internal_span("argon2id verify"): + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" +# Атрибуты спанов Redis: система и операция. Ключ (session:) и значения не пишутся. +REDIS_ATTRS = {"db.system.name": "redis", "server.address": "redis"} + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + with telemetry.client_span("redis SET", REDIS_ATTRS | {"db.operation.name": "SET"}): + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__, + extra={"event": "redis.unavailable", "error.type": type(exc).__name__}) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + with telemetry.client_span("redis GET", REDIS_ATTRS | {"db.operation.name": "GET"}): + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__, + extra={"event": "redis.unavailable", "error.type": type(exc).__name__}) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + with telemetry.client_span("redis DEL", REDIS_ATTRS | {"db.operation.name": "DEL"}): + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/broker.py b/broker.py new file mode 100644 index 0000000..9ec95f3 --- /dev/null +++ b/broker.py @@ -0,0 +1,104 @@ +"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms. + +Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует +scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure: +они только проверяют её пассивным declare и не могут создать/удалить очередь. + + exchange attachments (direct, durable) + --attachment.process.v1--> queue attachments.process.v1 (quorum, durable) + x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once), + x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря) + exchange attachments.dlx (direct, durable) + --attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py + через basic_get + nack не расходует delivery-limit) + +Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до +подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются. +""" +import json +import logging + +import pika +from pika.exceptions import AMQPError + +import pipeline +import settings + +log = logging.getLogger("messenger.broker") +logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал + +BrokerError = (AMQPError, OSError) + + +def parameters(url: str | None = None) -> pika.URLParameters: + params = pika.URLParameters(url or settings.BROKER_URL) + params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS + params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно + params.socket_timeout = 5 + params.connection_attempts = 1 + params.client_properties = {"connection_name": settings.INSTANCE_ID} + return params + + +def connect(url: str | None = None) -> pika.BlockingConnection: + return pika.BlockingConnection(parameters(url)) + + +def check_topology(channel) -> dict: + """Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях.""" + channel.exchange_declare(pipeline.EXCHANGE, passive=True) + main = channel.queue_declare(pipeline.QUEUE, passive=True).method + dlq = channel.queue_declare(pipeline.DLQ, passive=True).method + return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count} + + +def properties(message: dict) -> pika.BasicProperties: + # W3C Trace Context в native headers AMQP (основной carrier для worker) и в теле (поле схемы). + headers = {key: message[key] for key in ("traceparent", "tracestate") if key in message} + return pika.BasicProperties( + delivery_mode=pika.DeliveryMode.Persistent, + content_type="application/json", + message_id=f"{message['job_id']}:{message['attempt']}", + type=message["type"], + headers=headers or None, + ) + + +class Publisher: + """Канал в режиме confirm: basic_publish возвращает управление только после ack брокера. + + mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError. + Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях + строка outbox остаётся неопубликованной и будет отправлена позже. + """ + + def __init__(self): + self.connection = None + self.channel = None + + def open(self): + if self.channel is None or self.channel.is_closed or self.connection.is_closed: + self.close() + self.connection = connect() + self.channel = self.connection.channel() + self.channel.confirm_delivery() + check_topology(self.channel) + log.info("connected to broker, publisher confirms enabled", extra={"event": "broker.connected"}) + return self.channel + + def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY): + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True) + + def pump(self): + if self.connection is not None and self.connection.is_open: + self.connection.process_data_events(time_limit=0) # heartbeat в паузах + + def close(self): + try: + if self.connection is not None and self.connection.is_open: + self.connection.close() + except BrokerError: + pass + self.connection = None + self.channel = None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ecd322e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,494 @@ +name: messenger-lab6 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py. +x-pipeline-env: &pipeline-env + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + # Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults, + # доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0. + FAULT_INJECTION: ${FAULT_INJECTION:-1} + FAULTS_DIR: /faults + +# lab6: OpenTelemetry SDK всех процессов приложения -> OTel Collector (OTLP/HTTP, внутренняя сеть telemetry). +# Имена переменных — стандартные OTEL_* (их читает SDK); service.name задаётся в коде процесса и дублируется +# в OTEL_SERVICE_NAME у каждого сервиса. OTEL_SDK_DISABLED=true в .env выключает телеметрию целиком. +x-otel-env: &otel-env + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_TIMEOUT: "5" # секунды (Python SDK): экспорт не держит память дольше + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс + OTEL_BSP_MAX_QUEUE_SIZE: "2048" # спанов в памяти процесса, лишние отбрасываются и считаются + OTEL_BLRP_MAX_QUEUE_SIZE: "2048" # записей логов, так же + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" # otel.sdk.processor.*: очереди и потери SDK + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + +x-otel-depends: &otel-depends + # Порядок старта, но не жёсткая зависимость: без Collector приложение работает, телеметрия теряется. + condition: service_started + required: false + +x-alive-check: &alive-check + # Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад. + test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + +x-s3-api-secrets: &s3-api-secrets + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. + # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config + # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. + # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. + s3: + image: chrislusf/seaweedfs:3.80 + entrypoint: ["/usr/bin/weed"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=1024", "-volume.max=0", + "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: ["s3data:/data"] + networks: [data] + healthcheck: + # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») + test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 15s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. + s3-init: + build: *app-build + command: ["python", "s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL + OTEL_SERVICE_NAME: messenger-api + <<: [*s3-api-secrets, *otel-env] + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + redis: {condition: service_healthy} + otel-collector: *otel-depends + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. + janitor: + build: *app-build + command: ["python", "janitor.py"] + env_file: .env + environment: + CACHE_URL_FILE: "" # janitor не ходит в Redis + BROKER_URL_FILE: "" # и в брокер + OTEL_SERVICE_NAME: messenger-janitor + <<: [*s3-api-secrets, *otel-env] + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + otel-collector: *otel-depends + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + + # Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права — + # в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте. + # hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume. + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq + volumes: + - ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro + - rabbitmqdata:/var/lib/rabbitmq + secrets: [rabbitmq_definitions] + networks: [data] + healthcheck: + # Узел запущен, и vhost с очередями загружен (а не только «порт открыт»). + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"] + interval: 5s + timeout: 10s + retries: 24 + start_period: 20s + start_interval: 2s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает + # просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет. + dispatcher: + build: *app-build + command: ["python", "dispatcher.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_dispatcher + OTEL_SERVICE_NAME: messenger-dispatcher + <<: [*pipeline-env, *otel-env] + secrets: [database_url, broker_url_dispatcher] + volumes: ["faults:/faults"] + networks: [data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + otel-collector: *otel-depends + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + # Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем + # процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS. + # Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker. + worker: + build: *app-build + command: ["python", "worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_worker + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512} + OTEL_SERVICE_NAME: messenger-worker + <<: [*pipeline-env, *s3-api-secrets, *otel-env] + secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key] + volumes: ["faults:/faults"] + tmpfs: + - /tmp:size=256m,mode=1777 + networks: [data, telemetry] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + otel-collector: *otel-depends + healthcheck: *alive-check + restart: unless-stopped + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят; + # если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова. + stop_grace_period: 70s + deploy: + resources: + limits: {cpus: "2.00", memory: 1536M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # ---------------- lab6: наблюдаемость ---------------- + # OTel Collector: приём OTLP от приложения, memory_limiter + batch, ограниченные очереди exporter, + # отправка в Tempo / Loki / Prometheus. Не хранилище. Порты не публикуются (только сеть telemetry). + # Образ distroless (без shell), поэтому без healthcheck: здоровье видно по otelcol_* в Prometheus. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.164.0 + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro + networks: [telemetry] + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + # heap ограничивает memory_limiter (200 MiB); сверху — runtime Go и отображённый бинарник contrib. + limits: {cpus: "0.50", memory: 384M} + + # Одноразовый bootstrap прав на volumes Loki и Tempo (оба процесса работают как uid 10001, не root). + telemetry-init: + image: busybox:1.37.0 + user: root + command: ["sh", "-c", "chown -R 10001:10001 /loki /tempo"] + volumes: + - lokidata:/loki + - tempodata:/tempo + network_mode: none + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 32M} + + # Prometheus: метрики (OTLP receiver для Collector + pull служебных метрик), recording/alert rules. + # 127.0.0.1:9090 — только для evidence-скриптов и /alerts на защите. + prometheus: + image: prom/prometheus:v3.15.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --storage.tsdb.retention.size=1GB + - --web.enable-otlp-receiver + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - ./infra/prometheus/tests:/etc/prometheus/tests:ro # promtool test rules /etc/prometheus/tests/alerts_test.yml + - promdata:/prometheus + ports: + - "127.0.0.1:9090:9090" + networks: [telemetry, ops] + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Loki: логи (OTLP от Collector), файловое хранилище в volume lokidata. 127.0.0.1:3100 — для тестов. + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + ports: + - "127.0.0.1:3100:3100" + networks: [telemetry, ops] + depends_on: + telemetry-init: {condition: service_completed_successfully} + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Tempo: трассы (OTLP gRPC от Collector), блоки в volume tempodata. 127.0.0.1:3200 — для тестов. + tempo: + image: grafana/tempo:3.1.1 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + ports: + - "127.0.0.1:3200:3200" + networks: [telemetry, ops] + depends_on: + telemetry-init: {condition: service_completed_successfully} + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Grafana: интерфейс и запросы к трём хранилищам. Источники данных и dashboard — provisioning из Git + # (infra/grafana), ручного набора нет. Вход: admin / пароль из secrets/grafana_admin_password. + grafana: + image: grafana/grafana:12.4.14 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" # стенд не скачивает плагины из интернета при старте + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json + secrets: [grafana_admin_password] + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/etc/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + networks: [telemetry, ops] + depends_on: + prometheus: {condition: service_healthy} + loki: {condition: service_started} + tempo: {condition: service_started} + healthcheck: + # wget есть в образе на Alpine, curl — в варианте на Ubuntu: проверка работает в обоих. + test: ["CMD-SHELL", "wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health || curl -fsS -m 2 -o /dev/null http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 20s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 384M} + +networks: + edge: {} + data: + internal: true + # lab6: приложение -> Collector -> хранилища -> Grafana. Внутренняя: наружу ничего не публикуется. + telemetry: + internal: true + # Только для публикации UI/API хранилищ на 127.0.0.1 (internal-сеть не может публиковать порты). + ops: {} + +volumes: + pgdata: {} + redisdata: {} + s3data: {} + rabbitmqdata: {} + faults: {} + promdata: {} + lokidata: {} + tempodata: {} + grafanadata: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_config: {file: ./secrets/s3_config.json} + broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher} + broker_url_worker: {file: ./secrets/broker_url_worker} + rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json} + grafana_admin_password: {file: ./secrets/grafana_admin_password} diff --git a/db.py b/db.py new file mode 100644 index 0000000..96f57a9 --- /dev/null +++ b/db.py @@ -0,0 +1,452 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import pipeline +import settings +import telemetry +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +def check_connection(conn) -> None: + """Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса.""" + from opentelemetry.instrumentation.utils import suppress_instrumentation + with suppress_instrumentation(): + ConnectionPool.check_connection(conn) + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=check_connection, + open=False, +) + + +# lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»). +POOL_NAME = {"db.client.connection.pool.name": "messenger-api"} +POOL_WAIT = telemetry.meter.create_histogram( + "db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool") + + +def _pool_stats(): + try: + return pool.get_stats() + except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет + return None + + +def _observe_connections(_options): + from opentelemetry.metrics import Observation + stats = _pool_stats() + if not stats: + return [] + size, available = stats.get("pool_size", 0), stats.get("pool_available", 0) + return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}), + Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})] + + +def _observe_stat(key): + def callback(_options): + from opentelemetry.metrics import Observation + stats = _pool_stats() + return [Observation(stats.get(key, 0), POOL_NAME)] if stats else [] + return callback + + +telemetry.meter.create_observable_up_down_counter( + "db.client.connection.count", [_observe_connections], unit="{connection}", + description="Connections in the pool by state (used/idle)") +telemetry.meter.create_observable_up_down_counter( + "db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}", + description="Maximum pool size (DB_POOL_MAX)") +telemetry.meter.create_observable_up_down_counter( + "db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}", + description="Requests waiting for a free connection now") +telemetry.meter.create_observable_counter( + "db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}", + description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS") + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + started = time.perf_counter() + try: + with pool.connection(timeout=wait_seconds) as conn: + POOL_WAIT.record(time.perf_counter() - started, POOL_NAME) + yield conn + except PoolTimeout: + POOL_WAIT.record(time.perf_counter() - started, POOL_NAME) + raise + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__, + extra={"event": "db.unavailable", "error.type": type(exc).__name__}) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__, + extra={"event": "db.rejected_value", "error.type": type(exc).__name__}) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text", "filename") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" + + +def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() is not None + + +def check_member(chat_id: UUID, user_id: UUID) -> None: + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], + idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 + raise not_found("Chat not found") + if attachment_ids: + # Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409). + # FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние. + found = conn.execute( + """SELECT id, state FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""", + (attachment_ids, chat_id), + ).fetchall() + if len(found) != len(attachment_ids): + raise not_found("Attachment not found") + failed = any(row["state"] == "failed" for row in found) + else: + failed = False + saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m + WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""" + if failed: + # Повтор уже созданного сообщения возвращает прежний 201, даже если вложение + # с тех пор стало failed. Новое сообщение с failed-вложением — 409. + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if saved is None: + raise conflict("Failed attachment cannot be attached") + created = None + else: + created = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is None: + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if created is not None: + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES (%s, %s, %s, %s)""", + [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], + ) + return {**from_db(created), "attachment_ids": list(attachment_ids)} + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "m.chat_id = %s", [chat_id] + if after is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages m + WHERE {where} + ORDER BY m.created_at, m.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + +# ---------- вложения ---------- + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, " + "pipeline_version, created_at, updated_at, object_key") +VISIBLE_STATES = ("queued", "processing", "ready", "failed") + + +def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None: + """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging. + + Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт + и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД). + """ + try: + with connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, source_key, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", + (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, + width, height, source_key), + ) + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + + +def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None: + """Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией. + + После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL, + сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace). + """ + with connection() as conn: + with conn.transaction(): + row = conn.execute( + f"""UPDATE attachments SET state = 'queued', updated_at = now() + WHERE id = %s AND state = 'uploading' + RETURNING {ATTACHMENT_COLUMNS}""", + (attachment_id,), + ).fetchone() + if row is None: + return None + pipeline.enqueue_new(conn, attachment_id, actor) + return from_db(row) + + +def delete_upload_intent(attachment_id: UUID) -> None: + with connection(wait_seconds=1) as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) + + +def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: + """Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404. + + Возвращает строку и варианты текущей версии pipeline (они есть только у ready). + """ + with connection() as conn: + row = conn.execute( + f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} + FROM attachments a + JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s + WHERE a.id = %s AND a.state = ANY(%s)""", + (user_id, attachment_id, list(VISIBLE_STATES)), + ).fetchone() + if row is None: + raise not_found("Attachment not found") + variants = [] + if row["state"] == "ready": + variants = conn.execute( + """SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""", + (attachment_id, row["pipeline_version"]), + ).fetchall() + return {**from_db(row), "variants": variants} diff --git a/dispatcher.py b/dispatcher.py new file mode 100644 index 0000000..715d951 --- /dev/null +++ b/dispatcher.py @@ -0,0 +1,260 @@ +"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания. + +Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной +транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором +намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox +остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса, +ни ключей брокера, и недоступность брокера не мешает приёму файлов. + +Цикл: +1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP + LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT. + Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения, + который worker распознаёт по состоянию job (at-least-once, а не exactly-once). +2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed. +3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages). +Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру. +""" +import logging +import signal +import threading +import time +from pathlib import Path + +import psycopg +from opentelemetry.trace import SpanKind +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-dispatcher") + +import broker # noqa: E402 +import faults # noqa: E402 +import pipeline # noqa: E402 + +log = logging.getLogger("messenger.dispatcher") + +# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs, +# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому +# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик +# только читает последний снимок (не ходит в БД из потока экспорта). +STATS_SECONDS = 5 +STATS = {"at": 0.0, "db": None, "broker": None} +PUBLISHED = telemetry.meter.create_counter( + "messenger.messages.published", unit="{message}", + description="Outbox rows published to the broker (outcome: ok/error)") +QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE} + + +def _fresh(kind): + return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None + + +def _observe(kind, build): + def callback(_options): + from opentelemetry.metrics import Observation + data = _fresh(kind) + return [Observation(value, attrs) for value, attrs in build(data)] if data else [] + return callback + + +telemetry.meter.create_observable_gauge( + "messenger.queue.depth", [_observe("broker", lambda d: [ + (d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])], + unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)") +telemetry.meter.create_observable_gauge( + "messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])], + unit="{consumer}", description="Consumers subscribed to the queue") +telemetry.meter.create_observable_gauge( + "messenger.jobs.backlog", [_observe("db", lambda d: [ + (d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}), + (d["processing"], {"status": "processing"})])], + unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing") +telemetry.meter.create_observable_gauge( + "messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])], + unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits") +telemetry.meter.create_observable_gauge( + "messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])], + unit="{message}", description="Outbox rows due but not yet published") +telemetry.meter.create_observable_gauge( + "messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])], + unit="s", description="Age of the oldest due unpublished outbox row (publish lag)") + + +def collect_stats(conn, publisher) -> None: + """Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот.""" + if time.monotonic() - STATS["at"] < STATS_SECONDS: + return + db_stats = conn.execute( + """SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due, + count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed, + count(*) FILTER (WHERE status = 'processing') AS processing, + coalesce(extract(epoch FROM now() - min(available_at) + FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age + FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone() + outbox = conn.execute( + """SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due, + coalesce(extract(epoch FROM now() - min(available_at) + FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age + FROM outbox WHERE published_at IS NULL""").fetchone() + STATS["db"] = {**db_stats, **outbox} + try: + STATS["broker"] = broker.check_topology(publisher.open()) + except broker.BrokerError: + STATS["broker"] = None + publisher.close() + STATS["at"] = time.monotonic() + +STOP = threading.Event() +ACTOR = "dispatcher@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def publish_due(conn, publisher: broker.Publisher) -> int: + published = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, job_id, attempt, payload FROM outbox + WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", + (settings.DISPATCHER_BATCH,), + ).fetchall() + if not rows: + return published + for row in rows: + publish_one(conn, publisher, row) + published += 1 + log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)}) + if len(rows) < settings.DISPATCHER_BATCH: + return published + return published + + +def publish_one(conn, publisher: broker.Publisher, row: dict) -> None: + """Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст + уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу.""" + message = dict(row["payload"]) + job_id, attempt = message.get("job_id"), message.get("attempt") + attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send", + "messaging.destination.name": pipeline.EXCHANGE, + "messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY, + "messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id), + "messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]} + parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate")) + with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent, + kind=SpanKind.PRODUCER, attributes=attributes, + record_exception=False, set_status_on_exception=False) as span: + carrier = telemetry.inject() + if carrier.get("traceparent"): + message["traceparent"] = carrier["traceparent"] + message.pop("tracestate", None) + if carrier.get("tracestate"): + message["tracestate"] = carrier["tracestate"] + faults.crash("dispatcher_crash_before_publish") + try: + publisher.publish(message) + except broker.BrokerError as exc: + telemetry.mark_error(span, type(exc).__name__) + PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"}) + log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__, + extra={"event": "outbox.publish_failed", "job.id": str(job_id), + "error.type": type(exc).__name__}) + publisher.close() + raise + faults.crash("dispatcher_crash_after_confirm") + conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],)) + PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"}) + + +def note_failure(conn) -> None: + """Счётчик неудачных публикаций для наблюдения (ops.py status).""" + try: + conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1 + WHERE published_at IS NULL AND available_at <= now()""") + except psycopg.Error: + pass + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def run() -> None: + publisher = broker.Publisher() + conn = None + last_leases = last_reconcile = 0.0 + pause = 1.0 + while not STOP.is_set(): + try: + if conn is None or conn.closed: + conn = connect_db() + conn.execute("LISTEN outbox") + log.info("connected to PostgreSQL, listening for outbox notifications", + extra={"event": "db.connected"}) + touch_alive() + publish_due(conn, publisher) + collect_stats(conn, publisher) + now = time.monotonic() + if now - last_leases >= settings.DISPATCHER_POLL_SECONDS: + last_leases = now + if recovered := pipeline.recover_expired_leases(conn, ACTOR): + log.warning("recovered %s job(s) with expired lease", recovered, + extra={"event": "lease.recovered", "count": recovered}) + if now - last_reconcile >= 30: + last_reconcile = now + if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS): + log.warning("re-published %s queued job(s) without a message in flight", lost, + extra={"event": "outbox.republished", "count": lost}) + pause = 1.0 + publisher.pump() + # Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса. + for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1): + pass + except broker.BrokerError: + if conn is not None and not conn.closed: + note_failure(conn) + log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause, + extra={"event": "broker.unavailable"}) + STOP.wait(pause) + pause = min(pause * 2, 15) + except psycopg.Error as exc: + log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause, + extra={"event": "db.unavailable", "error.type": type(exc).__name__}) + try: + if conn is not None: + conn.close() + except psycopg.Error: + pass + conn = None + STOP.wait(pause) + pause = min(pause * 2, 15) + publisher.close() + if conn is not None: + conn.close() + log.info("stopped", extra={"event": "service.stopped"}) + telemetry.shutdown() + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID, + "ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"}) + run() + + +if __name__ == "__main__": + main() diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..cec62bc --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,123 @@ +# Runbooks алертов (lab6) + +Правила — `infra/prometheus/rules/alerts.yml`, unit-тесты правил — `infra/prometheus/tests/alerts_test.yml`. +Состояние алертов: http://127.0.0.1:9090/alerts или Grafana → Alerting → Alert rules (data source rules). +Dashboard: Grafana http://127.0.0.1:3000 → Messenger → «Messenger — lab6 observability». +Общий порядок: **метрика говорит «что и когда», трасса — «где именно», лог — «почему»**. + +Ссылки на трассу из лога: Grafana → Explore → Loki → запись → поле `trace_id` → «Открыть трассу». +Из спана в логи: Tempo → спан → «Logs for this span» (запрос по `trace_id`). + +--- + +## MessengerHighErrorRate + +**Что значит.** Больше 5% бизнес-запросов API (без `/health/*`) за минуту закончились 5xx, дольше минуты, +при трафике > 0,2 запроса/с. 4xx (неверный пароль, чужой чат) сюда не входят. + +**Влияние.** Пользователи получают 503/500, SLO-1 (99% без 5xx за час) расходует бюджет в 5+ раз быстрее нормы. + +**Найти причину.** +1. Какие маршруты и коды: dashboard «RPS по классу статуса» или + `sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))`. + 503 — недоступна зависимость, 500 — ошибка кода. +2. Медленная БД или медленный обработчик: + - `histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))` и + `db_client_connection_pending_requests` — запросы ждут соединение из пула → соединения заняты долгими запросами; + - Tempo, TraceQL `{ resource.service.name = "messenger-api" && status = error }` → открыть трассу: + если почти всё время server-спана занимает дочерний спан `INSERT`/`SELECT` (PostgreSQL) с ошибкой — + проблема в БД; если время в `argon2id hash`/`inspect image` или «пустое» время самого server-спана — в обработчике; + `redis GET` с ошибкой — Redis; `S3 PutObject` — SeaweedFS. +3. Логи API: `{service_name="messenger-api"} | event="db.unavailable"` (поле `error_type`: `QueryCanceled` — + statement_timeout 5 с, `PoolTimeout` — нет свободного соединения 3 с, `OperationalError` — БД недоступна); + `| event="redis.unavailable"`, `| event="s3.unavailable"`. +4. В PostgreSQL (если трассы указывают на БД): `docker compose exec postgres psql -U messenger -d messenger -c + "SELECT pid, state, wait_event_type, wait_event, now()-query_start AS age, left(query,60) FROM pg_stat_activity + WHERE datname='messenger' ORDER BY age DESC"` — блокировки (`wait_event_type = Lock`) и долгие запросы. + +**Исправить.** Снять блокировку/долгий запрос (`pg_terminate_backend(pid)`), восстановить зависимость +(`docker compose start postgres|redis|s3`). Не перезапускать API «на всякий случай»: причина не в нём. + +**Проверить восстановление.** Доля 5xx на dashboard → 0, алерт становится inactive через ~1–2 минуты +(окно rate 1m). Пример разбора: REPORT.md, инцидент 2. + +--- + +## MessengerQueueLagGrowing + +**Что значит.** Старейшее задание обработки вложения, которое уже можно брать (статус `queued`, +`available_at` прошёл), ждёт больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно. + +**Влияние.** Вложения остаются `queued`, клиенты не получают превью; SLO-3 (95% ready ≤ 30 с) нарушается. + +**Найти причину.** +1. `messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}` = 0 → worker не подключён + (см. MessengerWorkerNoConsumers). > 0 → worker есть, но не успевает или падает на заданиях. +2. Насыщение worker: `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` ≈ 1 и растёт + `messenger_queue_depth` → нужно больше потоков/реплик worker (`WORKER_CONCURRENCY`, `--scale worker=2`). +3. Ошибки обработки: `sum by (outcome) (rate(messenger_job_duration_seconds_count[5m]))` — много `retry`/`failed`; + логи `{service_name="messenger-worker"} | event=~"job.retry|job.failed"` (поле `error_type`). +4. Отличить от проблемы публикации: `messenger_outbox_oldest_age_seconds` большой → сообщения не доходят до + брокера (dispatcher/RabbitMQ), а не worker; лог `{service_name="messenger-dispatcher"} | event="broker.unavailable"`. +5. Трасса любой зависшей загрузки (по `trace_id` из лога API `event="http.request"` для POST attachments): + есть `publish attachments`, но нет `process attachments.process.v1` → сообщение в брокере, его никто не взял. + +**Исправить.** `docker compose start worker` (или `scripts/worker.sh start`); при насыщении — увеличить +параллелизм; при массовых ошибках — устранить их причину, затем `ops.py replay` для failed. + +**Проверить восстановление.** `messenger_jobs_backlog{status="queued"}` → 0, возраст → 0, вложения `ready`, +алерт inactive через ~15–30 с. Пример: REPORT.md, инцидент 1. + +--- + +## MessengerWorkerNoConsumers + +**Что значит.** RabbitMQ дольше минуты не видит ни одного подписчика очереди `attachments.process.v1`. + +**Найти причину.** `docker compose ps worker`; последние записи `{service_name="messenger-worker"}` в Loki +(`event="service.stopped"` — остановлен штатно; `event="broker.unavailable"` — не может подключиться к брокеру; +отсутствие записей и метрик `messenger_worker_capacity` — процесс не запущен). + +**Исправить.** Запустить worker / восстановить RabbitMQ. Принятые задания не теряются (outbox + durable очередь). + +--- + +## MessengerDbPoolSaturated + +**Что значит.** На реплике API запросы дольше минуты ждут свободное соединение из пула (`DB_POOL_MAX`=10). + +**Найти причину.** Почти всегда это следствие медленных запросов или блокировок в PostgreSQL, а не нехватки +соединений: те же шаги, что в MessengerHighErrorRate (трассы с долгими DB-спанами, `pg_stat_activity`). +Увеличивать пул имеет смысл, только если DB-спаны быстрые, а ожидание пула долгое при высоком RPS. + +--- + +## TelemetryExportFailing + +**Что значит.** Collector не может отправить данные в Tempo / Loki / Prometheus (метка `exporter`): очередь +exporter дольше минуты заполнена больше чем на 5% (хранилище не принимает), либо пакеты уже теряются — +у пакета истекло окно повторов 120 с (`send_failed`) или очередь переполнена (`enqueue_failed`). Окно повторов +считается для пакета с первой попытки, время ожидания в очереди в него не входит: очередь (200 пакетов) — +это и есть буфер на время недоступности. + +**Влияние.** На пользователей не влияет. Телеметрия этого периода частично потеряна (сколько — счётчики +`otelcol_exporter_send_failed_*`, `otelcol_exporter_enqueue_failed_*`). + +**Найти причину.** `docker compose ps tempo loki prometheus`; `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`; +логи Collector: `docker compose logs otel-collector --tail 50` (через Loki их не смотреть — это тот же конвейер). + +**Исправить.** `docker compose start tempo loki prometheus`. Очередь Collector досылается сама (в пределах окна +повторов). Проверить: `otelcol_exporter_queue_size` → 0, свежая трасса находится. Пример: REPORT.md, инцидент 3. + +--- + +## TelemetryCollectorDown + +**Что значит.** Prometheus не может забрать метрики Collector (`up{job="otel-collector"} == 0`). + +**Влияние.** Приложение работает; SDK каждого процесса держит не больше 2048 спанов и 2048 записей логов. +Потери считаются в самом процессе: пакет, который Collector не принял, — `otel_sdk_exporter_span_exported_total{error_type!=""}` +(обычно `ConnectionError`), переполнение очереди — `otel_sdk_processor_span_processed_total{error_type="queue_full"}`. Метрик приложения за +этот период нет (пропуск на графиках), cumulative-счётчики после восстановления продолжат расти без потери итогов. + +**Исправить.** `docker compose start otel-collector`; проверить, что трассы и метрики снова приходят. diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..a641476 --- /dev/null +++ b/errors.py @@ -0,0 +1,40 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..030c147 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-11-01 20:38:49, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_attachments.py +# exit code: 0, 10.3 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 23200a7c +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 0000000..6f5839a --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,18 @@ +# balancing: balancing, shared sessions/data/files, X-Instance-Id +# date: 2026-11-01 20:42:35, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 2.1 s + +20 x GET /users/me with one token -> replicas: {'1db2fedff78f': 10, '171a473b0d28': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 171a473b0d28; read back by ['171a473b0d28', '1db2fedff78f'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on 1db2fedff78f; old token then rejected by ['171a473b0d28', '1db2fedff78f'] +PASS: logout on one replica revokes the session on all of them +first POST on 1db2fedff78f -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae; same key on 171a473b0d28 -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +file uploaded via 1db2fedff78f; downloaded byte for byte via ['171a473b0d28', '1db2fedff78f'] +PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['171a473b0d28', '1db2fedff78f']; api containers: ['171a473b0d28', '1db2fedff78f'] +PASS: X-Instance-Id is in every response and points to a real container +6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..d9cad92 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-11-01 20:07:43, commit: 470d9d6 + uncommitted changes +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/configs.txt b/evidence/configs.txt new file mode 100644 index 0000000..add8075 --- /dev/null +++ b/evidence/configs.txt @@ -0,0 +1,32 @@ +# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning +# date: 2026-11-01 20:07:53, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/check_configs.py +# exit code: 0, 5.5 s + +--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0 + +--- promtool check config (infra/prometheus/prometheus.yml): exit 0 + +Checking /etc/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /etc/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool check rules (recording + alerts): exit 0 +Checking /etc/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /etc/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0 +SUCCESS +--- loki -verify-config (infra/loki/loki.yaml): exit 0 + +--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0 + +--- Grafana 12.4.14: database ok +datasource prometheus (prometheus): OK Successfully queried the Prometheus API. +datasource loki (loki): OK Data source successfully connected. +datasource tempo (tempo): OK Data source is working +dashboard 'Messenger — lab6 observability' in folder 'Messenger': 34 panels, provisioned=True, datasources ['loki', 'prometheus', 'tempo'] +PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana provisioned 3 healthy datasources and the dashboard without manual steps diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 0000000..f8ce011 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,47 @@ +# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage +# date: 2026-11-01 20:41:39, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_consistency.py +# exit code: 0, 159.9 s + +declared 9 000 000 bytes, sent 211817, closed the connection +declared 9 000 000 bytes, sent 3111817, closed the connection +rows in attachments: 63 -> 63; objects in bucket: 240 -> 240 +temporary upload files in api containers: {'171a473b0d28': 0, '1db2fedff78f': 0} (removed when the request ends) +api containers have no volumes: the original is never stored locally +PASS: an interrupted upload leaves no row, no object and no local file +crash after S3 PUT: row b1065ae1 state=uploading, object staging/b5763f64729a4b8a8c899bcd7009e599 exists in S3 +the half-written attachment is invisible: GET -> 404, attach to a message -> 404 +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.133} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.132} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.101} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.134} -> idempotent +the ready attachment of the same chat is untouched +PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period +object staging/23223961df3644dbbaa0fa4f40d57b82 put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.103} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.123} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 0, "seconds": 0.134} -> orphan deleted +ready attachments older than the grace period are untouched (they have rows) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=1 and keeps the row; download -> 503 (not 500) +PASS: orphans are found and deleted only after the grace period; missing objects are reported +--- S3 stopped (connection refused / name not resolved): docker compose stop s3 +upload -> 503 unavailable in 20234 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose start s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 259, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.142} +PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up +--- S3 frozen (no answer, timeouts): docker compose pause s3 +upload -> 503 unavailable in 14703 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose unpause s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 267, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.192} +PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up +5 passed, 0 failed diff --git a/evidence/dashboard.txt b/evidence/dashboard.txt new file mode 100644 index 0000000..e0daa60 --- /dev/null +++ b/evidence/dashboard.txt @@ -0,0 +1,36 @@ +# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels +# date: 2026-11-01 20:09:12, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/check_dashboard.py +# exit code: 0, 1.0 s + +PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident) + empty now: CPU контейнеров / лимит + empty now: Collector: потери и отказы + empty now: SDK приложений: экспорт и потери + empty now: alerts.yml: MessengerDbPoolSaturated + empty now: alerts.yml: MessengerHighErrorRate + empty now: alerts.yml: MessengerQueueLagGrowing + empty now: alerts.yml: MessengerWorkerNoConsumers + empty now: alerts.yml: TelemetryCollectorDown + empty now: alerts.yml: TelemetryExportFailing + empty now: Алерты Prometheus (pending/firing) + empty now: Память: RSS процесса и контейнер +LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error" +TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 4 trace(s) + +data: http_server_request_duration_seconds_count{job="messenger-api"} 48 series <- HTTP requests/duration (3) +data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3) +data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4) +data: messenger_queue_depth 2 series <- queue depth (4) +data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4) +data: db_client_connection_count 4 series <- DB pool used/idle (4) +data: db_client_connection_pending_requests 2 series <- DB pool waiting (4) +data: messenger_worker_busy 1 series <- worker busy (4) +data: messenger_worker_capacity 1 series <- worker capacity (4) +data: process_cpu_time_seconds_total 12 series <- process CPU (4) +data: process_memory_usage_bytes 6 series <- process RAM (4) +data: otelcol_exporter_queue_size 3 series <- collector queue (4) + +labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version +http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready'] +PASS: every dashboard/rule query runs, key signals have data, no forbidden labels diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 0000000..91eca92 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-11-01 20:42:53, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 11.6 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 71 -> 72 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 10; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 0000000..49f2722 --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,109 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 68448 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.007..0.019 rows=51 loops=1) + Buffers: shared hit=54 + -> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1) + Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Buffers: shared hit=54 + Planning: + Buffers: shared hit=8 + Planning Time: 0.067 ms + Execution Time: 0.026 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.007..0.012 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid))) + Buffers: shared hit=5 + Planning Time: 0.083 ms + Execution Time: 0.047 ms + + +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.024..0.176 rows=51 loops=1) + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1) + Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid))) + Buffers: shared hit=59 + SubPlan 1 + -> Sort (actual time=0.002..0.002 rows=0 loops=51) + Sort Key: a."position" + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Recheck Cond: (message_id = m.id) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Index Cond: (message_id = m.id) + Buffers: shared hit=51 + Planning: + Buffers: shared hit=63 + Planning Time: 0.210 ms + Execution Time: 0.193 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +-------------------------------------------------------------------------------- + Limit (actual time=4.087..4.090 rows=51 loops=1) + Buffers: shared hit=1255 + -> Sort (actual time=3.670..3.904 rows=10051 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 2377kB + Buffers: shared hit=1255 + -> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1) + Filter: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Rows Removed by Filter: 53448 + Buffers: shared hit=1255 + Planning Time: 0.041 ms + Execution Time: 4.136 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.375..1.378 rows=51 loops=1) + Buffers: shared hit=1006 + -> Sort (actual time=1.374..1.376 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=1006 + -> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1) + Recheck Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Filter: (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid)) + Rows Removed by Filter: 10000 + Heap Blocks: exact=991 + Buffers: shared hit=1006 + -> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1) + Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) + Buffers: shared hit=15 + Planning: + Buffers: shared hit=4 + Planning Time: 0.057 ms + Execution Time: 1.395 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..1824deb --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,46 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-11-01 20:42:57, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 1.2 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=54 + -> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1) + Buffers: shared hit=54 + Buffers: shared hit=8 + Execution Time: 0.026 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Buffers: shared hit=5 + Execution Time: 0.047 ms +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1) + Buffers: shared hit=59 + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + Buffers: shared hit=63 + Execution Time: 0.193 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=1255 + Buffers: shared hit=1255 + -> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1) + Buffers: shared hit=1255 + Execution Time: 4.136 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=1006 + Buffers: shared hit=1006 + -> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1) + Buffers: shared hit=1006 + -> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1) + Buffers: shared hit=15 + Buffers: shared hit=4 + Execution Time: 1.395 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..8fb5f43 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica, file downloaded via the other one (make test-failover) +# date: 2026-11-01 20:43:39, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 42.0 s + +hook: scripts\stop-one.cmd; api replicas: ['171a473b0d28', '1db2fedff78f'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Up 41 seconds (healthy) +api 1db2fedff78f Up 41 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.75} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Exited (0) 11 seconds ago +api 1db2fedff78f Up 58 seconds (healthy) +the hook stopped 171a473b0d28 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 171a473b0d28 -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Up 2 seconds (healthy) +api 1db2fedff78f Up About a minute (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.578} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 171a473b0d28 Up 19 seconds (healthy) +api 1db2fedff78f Exited (0) 11 seconds ago +the hook stopped 1db2fedff78f (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 1db2fedff78f -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 0000000..3f08921 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-11-01 20:45:06, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 55.6 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (8016 ms) +valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1016 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..63cdb70 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-11-01 20:42:37, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.2 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'171a473b0d28': 10, '1db2fedff78f': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/incident-collector-down.json b/evidence/incident-collector-down.json new file mode 100644 index 0000000..c3d183a --- /dev/null +++ b/evidence/incident-collector-down.json @@ -0,0 +1,39 @@ +{ + "scenario": "collector-down", + "date": "2026-11-01 20:25:18", + "outage_s": 90, + "baseline": { + "requests": 870, + "codes": { + "201": 870 + }, + "success_ratio": 1.0, + "success_rps": 58.0, + "p95_ms_successful": 32.0 + }, + "during": { + "requests": 5852, + "codes": { + "201": 5852 + }, + "success_ratio": 1.0, + "success_rps": 57.9, + "p95_ms_successful": 31.0 + }, + "api_mib_before": 295.2, + "api_mib_max": 313.5, + "sdk_lost_spans_api": { + "processor_queue_full": 8252, + "exporter_failed": 10752 + }, + "sdk_exported_ok_after_restart": 10287, + "after": { + "requests": 1403, + "codes": { + "201": 1403 + }, + "success_ratio": 1.0, + "success_rps": 56.1, + "p95_ms_successful": 32.0 + } +} \ No newline at end of file diff --git a/evidence/incident-collector.txt b/evidence/incident-collector.txt new file mode 100644 index 0000000..87b1dbb --- /dev/null +++ b/evidence/incident-collector.txt @@ -0,0 +1,20 @@ +# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted +# date: 2026-11-01 20:27:53, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py collector-down +# exit code: 0, 155.8 s + +=== collector-down +[ 24.6s] baseline: {'requests': 870, 'codes': {'201': 870}, 'success_ratio': 1.0, 'success_rps': 58.0, 'p95_ms_successful': 32.0}, api containers 295.2 MiB +[ 25.3s] docker compose stop otel-collector +[ 29.3s] {'t': 0.7, 'api_mib': 292.8} +[ 43.6s] {'t': 14.7, 'api_mib': 298.9} +[ 58.0s] {'t': 29.0, 'api_mib': 306.5} +[ 72.6s] {'t': 43.4, 'api_mib': 307.8} +[ 87.0s] {'t': 58.0, 'api_mib': 308.2} +[ 101.0s] {'t': 72.4, 'api_mib': 313.5} +[ 115.0s] {'t': 86.4, 'api_mib': 313.3} +[ 125.6s] docker compose start otel-collector +[ 155.6s] API during collector outage: {'requests': 5852, 'codes': {'201': 5852}, 'success_ratio': 1.0, 'success_rps': 57.9, 'p95_ms_successful': 31.0}; api memory 295.2 -> max 313.5 MiB; SDK lost spans (api): {'processor_queue_full': 8252, 'exporter_failed': 10752}; exported OK in the last minute after restart: 10287 +[ 155.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-collector-down.json +PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start +1 passed, 0 failed diff --git a/evidence/incident-db-slow.json b/evidence/incident-db-slow.json new file mode 100644 index 0000000..2bc2919 --- /dev/null +++ b/evidence/incident-db-slow.json @@ -0,0 +1,82 @@ +{ + "scenario": "db-slow", + "date": "2026-11-01 20:13:54", + "lock_seconds": 210, + "baseline": { + "requests": 2310, + "codes": { + "201": 2310 + }, + "success_ratio": 1.0, + "success_rps": 115.5, + "p95_ms_successful": 32.0, + "server_p95_s": 0.019 + }, + "firing_after_lock_s": 140.8, + "diagnosis": { + "http_5xx_ratio_1m": 1.0, + "http_p95_messages_s": 7.375, + "pool_used": 8.0, + "pool_wait_p95_s": 0.001, + "status_codes_1m": { + "201": 0.0, + "200": 0.0, + "404": 0.0, + "400": 0.0, + "409": 0.0, + "503": 1.6 + }, + "reads_during_lock_ms": { + "GET /users/me + GET /chats, avg per request": 17.2, + "max": 23.5 + }, + "api_log_db_unavailable_by_error_type": { + "QueryCanceled": 216 + }, + "error_trace": { + "trace_id": "237e2d6b1c7db22adbb7d4752fa47f2", + "server_ms": 5013.3, + "slowest_db_span": "INSERT", + "slowest_db_ms": 5001.3, + "db_span_status": "ERROR", + "db_share": 0.998, + "tree": [ + "- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503", + " - [api] redis GET (CLIENT, 1.1 ms)", + " - [api] SELECT (CLIENT, 1.6 ms)", + " - [api] INSERT (CLIENT, 5001.3 ms) ERROR" + ] + }, + "handler_cpu_contrast": { + "trace_id": "4a222e478c2d51006538a7bfeb485eb1", + "server_ms": 21.2, + "argon2id_ms": 16.4, + "db_ms_total": 0.7, + "tree": [ + "- [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201", + " - [api] argon2id hash (INTERNAL, 16.4 ms)", + " - [api] INSERT (CLIENT, 0.7 ms)" + ] + } + }, + "during_lock": { + "requests": 328, + "codes": { + "503": 320, + "201": 8 + }, + "success_ratio": 0.0244, + "success_rps": 0.0, + "p95_ms_successful": 2156.0 + }, + "resolved_after_unlock_s": 50.1, + "after": { + "requests": 5269, + "codes": { + "201": 5269 + }, + "success_ratio": 1.0, + "success_rps": 116.8, + "p95_ms_successful": 31.0 + } +} \ No newline at end of file diff --git a/evidence/incident-db.txt b/evidence/incident-db.txt new file mode 100644 index 0000000..a5e8378 --- /dev/null +++ b/evidence/incident-db.txt @@ -0,0 +1,64 @@ +# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved +# date: 2026-11-01 20:18:39, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py db-slow +# exit code: 0, 285.7 s + +=== db-slow +[ 0.0s] load: 8 threads POST /messages +[ 25.0s] baseline: {'requests': 2310, 'codes': {'201': 2310}, 'success_ratio': 1.0, 'success_rps': 115.5, 'p95_ms_successful': 32.0, 'server_p95_s': 0.019} +[ 25.0s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait) +[ 25.1s] 5xx ratio=None p95=0.019 s pool used=0.0 waiting=0.0 +[ 30.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0 +[ 35.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0 +[ 40.1s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0 +[ 45.2s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0 +[ 50.2s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0 +[ 55.3s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0 +[ 60.3s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 65.3s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0 +[ 70.4s] 5xx ratio=0.0 p95=0.045 s pool used=8.0 waiting=0.0 +[ 75.4s] 5xx ratio=0.015 p95=7.18 s pool used=8.0 waiting=0.0 +[ 80.4s] 5xx ratio=0.015 p95=7.234 s pool used=8.0 waiting=0.0 +[ 85.4s] 5xx ratio=0.015 p95=7.375 s pool used=8.0 waiting=0.0 +[ 90.4s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0 +[ 95.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0 +[ 100.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0 +[ 105.5s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 110.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 115.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 120.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 125.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 130.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 135.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 140.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 145.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 150.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 155.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 160.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 165.8s] ALERT MessengerHighErrorRate firing (140.8 s after lock) +[ 172.2s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 17.2, "max": 23.5}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 216}} +[ 172.2s] error trace (DB): {"trace_id": "237e2d6b1c7db22adbb7d4752fa47f2", "server_ms": 5013.3, "slowest_db_span": "INSERT", "slowest_db_ms": 5001.3, "db_span_status": "ERROR", "db_share": 0.998} + - [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503 + - [api] redis GET (CLIENT, 1.1 ms) + - [api] SELECT (CLIENT, 1.6 ms) + - [api] INSERT (CLIENT, 5001.3 ms) ERROR +[ 172.2s] contrast, slow handler (CPU): {"trace_id": "4a222e478c2d51006538a7bfeb485eb1", "server_ms": 21.2, "argon2id_ms": 16.4, "db_ms_total": 0.7} + - [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201 + - [api] argon2id hash (INTERNAL, 16.4 ms) + - [api] INSERT (CLIENT, 0.7 ms) +[ 235.5s] lock released (psql exit 0), 210.5 s after LOCK +[ 235.5s] ... MessengerHighErrorRate: firing (0 s) +[ 240.5s] ... MessengerHighErrorRate: firing (5 s) +[ 245.5s] ... MessengerHighErrorRate: firing (10 s) +[ 250.5s] ... MessengerHighErrorRate: firing (15 s) +[ 255.5s] ... MessengerHighErrorRate: firing (20 s) +[ 260.6s] ... MessengerHighErrorRate: firing (25 s) +[ 265.6s] ... MessengerHighErrorRate: firing (30 s) +[ 270.6s] ... MessengerHighErrorRate: firing (35 s) +[ 275.6s] ... MessengerHighErrorRate: firing (40 s) +[ 280.6s] ... MessengerHighErrorRate: firing (45 s) +[ 285.6s] ALERT MessengerHighErrorRate resolved 50.1 s after unlock +[ 285.6s] load: during lock {'requests': 328, 'codes': {'503': 320, '201': 8}, 'success_ratio': 0.0244, 'success_rps': 0.0, 'p95_ms_successful': 2156.0}; after {'requests': 5269, 'codes': {'201': 5269}, 'success_ratio': 1.0, 'success_rps': 116.8, 'p95_ms_successful': 31.0} +[ 285.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-db-slow.json +PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved +1 passed, 0 failed diff --git a/evidence/incident-telemetry-down.json b/evidence/incident-telemetry-down.json new file mode 100644 index 0000000..40321f2 --- /dev/null +++ b/evidence/incident-telemetry-down.json @@ -0,0 +1,273 @@ +{ + "scenario": "telemetry-down", + "date": "2026-11-01 20:18:47", + "stopped": [ + "tempo", + "loki" + ], + "outage_s": 150, + "baseline": { + "requests": 899, + "codes": { + "201": 899 + }, + "success_ratio": 1.0, + "success_rps": 59.9, + "p95_ms_successful": 31.0 + }, + "collector_heap_before_mib": 35.5, + "collector_container_before_mib": 75.9, + "during": { + "requests": 9555, + "codes": { + "201": 9555 + }, + "success_ratio": 1.0, + "success_rps": 59.6, + "p95_ms_successful": 31.0 + }, + "samples": [ + { + "t": 0.0, + "collector_heap_mib": 56.4, + "collector_rss_mib": 237.6, + "collector_container_mib": 76.0, + "queue": { + "otlp_http/loki": 12.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 5.0 + }, + "alert": "inactive" + }, + { + "t": 12.1, + "collector_heap_mib": 45.4, + "collector_rss_mib": 237.8, + "collector_container_mib": 76.0, + "queue": { + "otlp_http/loki": 20.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 9.0 + }, + "alert": "pending" + }, + { + "t": 24.2, + "collector_heap_mib": 54.2, + "collector_rss_mib": 237.8, + "collector_container_mib": 76.0, + "queue": { + "otlp_http/loki": 27.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 12.0 + }, + "alert": "pending" + }, + { + "t": 36.2, + "collector_heap_mib": 54.2, + "collector_rss_mib": 237.8, + "collector_container_mib": 78.1, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 15.0 + }, + "alert": "pending" + }, + { + "t": 48.7, + "collector_heap_mib": 62.9, + "collector_rss_mib": 240.4, + "collector_container_mib": 82.2, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 15.0 + }, + "alert": "pending" + }, + { + "t": 60.8, + "collector_heap_mib": 51.2, + "collector_rss_mib": 244.4, + "collector_container_mib": 82.2, + "queue": { + "otlp_http/loki": 42.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 19.0 + }, + "alert": "pending" + }, + { + "t": 73.1, + "collector_heap_mib": 59.9, + "collector_rss_mib": 244.4, + "collector_container_mib": 82.3, + "queue": { + "otlp_http/loki": 50.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 22.0 + }, + "alert": "firing" + }, + { + "t": 85.2, + "collector_heap_mib": 70.5, + "collector_rss_mib": 248.4, + "collector_container_mib": 88.7, + "queue": { + "otlp_http/loki": 57.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 26.0 + }, + "alert": "firing" + }, + { + "t": 97.7, + "collector_heap_mib": 70.5, + "collector_rss_mib": 248.4, + "collector_container_mib": 94.5, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 27.0 + }, + "alert": "firing" + }, + { + "t": 109.8, + "collector_heap_mib": 56.3, + "collector_rss_mib": 256.7, + "collector_container_mib": 94.5, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 27.0 + }, + "alert": "firing" + }, + { + "t": 122.1, + "collector_heap_mib": 67.5, + "collector_rss_mib": 256.7, + "collector_container_mib": 94.5, + "queue": { + "otlp_http/loki": 70.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 31.0 + }, + "alert": "firing" + }, + { + "t": 134.2, + "collector_heap_mib": 76.1, + "collector_rss_mib": 256.9, + "collector_container_mib": 98.8, + "queue": { + "otlp_http/loki": 78.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 34.0 + }, + "alert": "firing" + }, + { + "t": 146.2, + "collector_heap_mib": 86.9, + "collector_rss_mib": 267.1, + "collector_container_mib": 107.0, + "queue": { + "otlp_http/loki": 85.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 38.0 + }, + "alert": "firing" + } + ], + "collector_heap_max_mib": 86.9, + "collector_rss_max_mib": 267.1, + "collector_container_max_mib": 107.0, + "TelemetryExportFailing_firing_at_s": 73.1, + "queue_drained_after_start_s": 60.1, + "probe_traces": [ + { + "sent_at_s": 0.0, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 12.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 24.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 36.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 48.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 60.8, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 73.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 85.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 97.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 109.8, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 122.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 134.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 146.2, + "http": 201, + "trace_in_tempo": true + } + ], + "lost_in_collector": { + "spans": 4096, + "log_records": 235 + }, + "resolved_after_start_s": 165.6, + "after": { + "requests": 9370, + "codes": { + "201": 9370 + }, + "success_ratio": 1.0, + "success_rps": 58.3, + "p95_ms_successful": 31.0 + } +} \ No newline at end of file diff --git a/evidence/incident-telemetry.txt b/evidence/incident-telemetry.txt new file mode 100644 index 0000000..32e937c --- /dev/null +++ b/evidence/incident-telemetry.txt @@ -0,0 +1,47 @@ +# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted +# date: 2026-11-01 20:25:06, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py telemetry-down +# exit code: 0, 379.8 s + +=== telemetry-down +[ 22.5s] baseline: {'requests': 899, 'codes': {'201': 899}, 'success_ratio': 1.0, 'success_rps': 59.9, 'p95_ms_successful': 31.0}, collector heap 35.5 MiB, container 75.9 MiB +[ 53.8s] docker compose stop tempo loki: stopped +[ 55.9s] {'t': 0.0, 'collector_heap_mib': 56.4, 'collector_rss_mib': 237.6, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 5.0}, 'alert': 'inactive'} +[ 68.0s] {'t': 12.1, 'collector_heap_mib': 45.4, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 9.0}, 'alert': 'pending'} +[ 80.0s] {'t': 24.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 12.0}, 'alert': 'pending'} +[ 92.5s] {'t': 36.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 78.1, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'} +[ 104.5s] {'t': 48.7, 'collector_heap_mib': 62.9, 'collector_rss_mib': 240.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'} +[ 116.9s] {'t': 60.8, 'collector_heap_mib': 51.2, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'} +[ 128.9s] {'t': 73.1, 'collector_heap_mib': 59.9, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.3, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'firing'} +[ 141.5s] {'t': 85.2, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 88.7, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'firing'} +[ 153.5s] {'t': 97.7, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'} +[ 165.9s] {'t': 109.8, 'collector_heap_mib': 56.3, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'} +[ 178.0s] {'t': 122.1, 'collector_heap_mib': 67.5, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 70.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 31.0}, 'alert': 'firing'} +[ 190.0s] {'t': 134.2, 'collector_heap_mib': 76.1, 'collector_rss_mib': 256.9, 'collector_container_mib': 98.8, 'queue': {'otlp_http/loki': 78.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 34.0}, 'alert': 'firing'} +[ 202.6s] {'t': 146.2, 'collector_heap_mib': 86.9, 'collector_rss_mib': 267.1, 'collector_container_mib': 107.0, 'queue': {'otlp_http/loki': 85.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 38.0}, 'alert': 'firing'} +[ 214.0s] docker compose start tempo loki +[ 274.1s] exporter queues drained 60.1 s after start +[ 289.4s] ... TelemetryExportFailing: firing (0 s) +[ 294.4s] ... TelemetryExportFailing: firing (5 s) +[ 299.5s] ... TelemetryExportFailing: firing (10 s) +[ 304.5s] ... TelemetryExportFailing: firing (15 s) +[ 309.5s] ... TelemetryExportFailing: firing (20 s) +[ 314.5s] ... TelemetryExportFailing: firing (25 s) +[ 319.5s] ... TelemetryExportFailing: firing (30 s) +[ 324.5s] ... TelemetryExportFailing: firing (35 s) +[ 329.5s] ... TelemetryExportFailing: firing (40 s) +[ 334.5s] ... TelemetryExportFailing: firing (45 s) +[ 339.6s] ... TelemetryExportFailing: firing (50 s) +[ 344.6s] ... TelemetryExportFailing: firing (55 s) +[ 349.6s] ... TelemetryExportFailing: firing (60 s) +[ 354.6s] ... TelemetryExportFailing: firing (65 s) +[ 359.6s] ... TelemetryExportFailing: firing (70 s) +[ 364.6s] ... TelemetryExportFailing: firing (75 s) +[ 369.6s] ... TelemetryExportFailing: firing (80 s) +[ 374.7s] ... TelemetryExportFailing: firing (85 s) +[ 379.7s] API during outage: {'requests': 9555, 'codes': {'201': 9555}, 'success_ratio': 1.0, 'success_rps': 59.6, 'p95_ms_successful': 31.0} +[ 379.7s] probe traces: 13/13 reached Tempo (lost: sent at [] s) +[ 379.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 235}; collector heap 35.5 -> max 86.9 MiB (memory_limiter 200 MiB), container max 107.0 MiB +[ 379.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-telemetry-down.json +PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved +1 passed, 0 failed diff --git a/evidence/incident-worker-stop.json b/evidence/incident-worker-stop.json new file mode 100644 index 0000000..d1b90eb --- /dev/null +++ b/evidence/incident-worker-stop.json @@ -0,0 +1,50 @@ +{ + "scenario": "worker-stop", + "date": "2026-11-01 20:09:59", + "firing_after_stop_s": { + "MessengerWorkerNoConsumers": 101.7, + "MessengerQueueLagGrowing": 131.8 + }, + "diagnosis": { + "queue_consumers": 0.0, + "broker_queue_depth": 6.0, + "jobs_backlog_queued": 3.0, + "oldest_job_age_s": 127.3, + "outbox_oldest_age_s": 0.0, + "published_ok_last_5m": 16.41239671427261, + "worker_metrics_last_seen_s": 131.0, + "last_worker_log": { + "event": "service.stopped", + "message": "stopped" + }, + "upload_trace_span_names": [ + "INSERT", + "NOTIFY", + "POST /api/v1/chats/{chat_id}/attachments", + "S3 PutObject", + "SELECT", + "UPDATE", + "create attachments", + "inspect image", + "publish attachments", + "redis GET", + "republish lost message" + ], + "upload_trace_has_consumer_span": false + }, + "ready_after_start_s": [ + 4.2, + 4.2, + 4.7 + ], + "resolved_after_start_s": { + "MessengerWorkerNoConsumers": 29.7, + "MessengerQueueLagGrowing": 29.7 + }, + "after_recovery": { + "jobs_backlog_queued": 0.0, + "oldest_job_age_s": 0.0, + "queue_consumers": 2.0 + }, + "time_to_ready_p95_5m_s": 282.0 +} \ No newline at end of file diff --git a/evidence/incident-worker.txt b/evidence/incident-worker.txt new file mode 100644 index 0000000..b156c4a --- /dev/null +++ b/evidence/incident-worker.txt @@ -0,0 +1,49 @@ +# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved +# date: 2026-11-01 20:12:40, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/incidents.py worker-stop +# exit code: 0, 161.8 s + +=== worker-stop +[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s +[ 1.2s] docker compose stop worker +[ 1.3s] uploaded 3 attachments: 202 queued (API accepts without worker) +[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {} +[ 6.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {} +[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {} +[ 16.5s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {} +[ 21.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {} +[ 26.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {} +[ 31.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {} +[ 36.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {} +[ 41.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 46.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 51.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 56.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 61.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 66.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 71.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 76.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 81.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 86.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 91.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 96.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 101.8s] ALERT MessengerWorkerNoConsumers firing (101.7 s after stop) +[ 101.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 106.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 111.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 116.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 121.8s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 126.9s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 131.9s] ALERT MessengerQueueLagGrowing firing (131.8 s after stop) +[ 132.0s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 6.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 127.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 16.41239671427261, "worker_metrics_last_seen_s": 131.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET", "republish lost message"], "upload_trace_has_consumer_span": false} +[ 135.3s] docker compose start worker +[ 136.6s] all attachments ready 4.7 s after worker start; backlog now 3.0 +[ 136.6s] ... MessengerWorkerNoConsumers: firing (0 s) +[ 141.6s] ... MessengerWorkerNoConsumers: firing (5 s) +[ 146.6s] ... MessengerWorkerNoConsumers: firing (10 s) +[ 151.7s] ... MessengerWorkerNoConsumers: firing (15 s) +[ 156.7s] ... MessengerWorkerNoConsumers: firing (20 s) +[ 161.7s] alerts resolved: {'MessengerWorkerNoConsumers': 29.7, 'MessengerQueueLagGrowing': 29.7}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0} +[ 161.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-worker-stop.json +PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved +1 passed, 0 failed diff --git a/evidence/jobs-concurrency.txt b/evidence/jobs-concurrency.txt new file mode 100644 index 0000000..87f2616 --- /dev/null +++ b/evidence/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-11-01 20:38:38, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py concurrency +# exit code: 0, 15.8 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) +1 passed, 0 failed diff --git a/evidence/jobs-crash.txt b/evidence/jobs-crash.txt new file mode 100644 index 0000000..eade149 --- /dev/null +++ b/evidence/jobs-crash.txt @@ -0,0 +1,15 @@ +# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease +# date: 2026-11-01 20:33:58, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease +# exit code: 0, 157.0 s + +--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат. +worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] +--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack. +worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) +--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit). +staging source still present after the crash (worker died before cleanup); waiting 60 s window +janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone +--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; +frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] +4 passed, 0 failed diff --git a/evidence/jobs-janitor-replay.txt b/evidence/jobs-janitor-replay.txt new file mode 100644 index 0000000..800d25f --- /dev/null +++ b/evidence/jobs-janitor-replay.txt @@ -0,0 +1,13 @@ +# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix) +# date: 2026-11-01 20:38:23, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_janitor_replay.py +# exit code: 0, 187.0 s + +race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +waiting 86 s until the leftovers are older than grace (120 s) +janitor pass over the bucket: did not try to delete any object of the replayable failed attachment +ops replay -> job 09f5cf6c-bca3-4759-9930-8d18968ed1be ready +after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 0, deleted 0 +retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)" +1 passed, 0 failed diff --git a/evidence/jobs-limits.txt b/evidence/jobs-limits.txt new file mode 100644 index 0000000..8fd987d --- /dev/null +++ b/evidence/jobs-limits.txt @@ -0,0 +1,11 @@ +# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) +# date: 2026-11-01 20:30:08, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py limits +# exit code: 0, 72.7 s + +--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker. +10 MiB + 8 bytes -> 413 before any job is created +imaging.py on 8000x5000 PNG: exit 0, peak RSS 226.9 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large) +memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived +imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s +1 passed, 0 failed diff --git a/evidence/jobs-outbox.txt b/evidence/jobs-outbox.txt new file mode 100644 index 0000000..15d44c4 --- /dev/null +++ b/evidence/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-11-01 20:31:21, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py outbox-crash +# exit code: 0, 17.2 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/jobs-recovery.txt b/evidence/jobs-recovery.txt new file mode 100644 index 0000000..ff53686 --- /dev/null +++ b/evidence/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-11-01 20:31:04, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 55.2 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment 0d7622f1-7c85-4c52-9dc0-2dad8f729c8d +message with attachment 201, content 409, other messages 201; job 6a45774d-0d88-46ec-87b9-70fb4c122bbd published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job 34eb3c30-1942-4777-bce2-1ea2a4eca4c9 +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/jobs-retry.txt b/evidence/jobs-retry.txt new file mode 100644 index 0000000..7dcf40b --- /dev/null +++ b/evidence/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-11-01 20:35:16, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 77.6 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] +4 passed, 0 failed diff --git a/evidence/jobs-variants.txt b/evidence/jobs-variants.txt new file mode 100644 index 0000000..7a843a4 --- /dev/null +++ b/evidence/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-11-01 20:28:56, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_jobs.py variants +# exit code: 0, 20.1 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..ae3d0d2 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 4, + "successful_requests": 7178, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 239.02, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 47.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..01637d0 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-11-01 20:44:10, commit: 470d9d6 + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.3 s + +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 4, + "successful_requests": 7178, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 239.02, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 47.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/attachments.txt b/evidence/native-sandbox/attachments.txt new file mode 100644 index 0000000..516518a --- /dev/null +++ b/evidence/native-sandbox/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-11-01 00:52:47, commit: unknown +# command: python student-tests/test_attachments.py +# exit code: 0, 13.5 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f30c028f +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (70 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (84 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/native-sandbox/check.txt b/evidence/native-sandbox/check.txt new file mode 100644 index 0000000..6bb54d6 --- /dev/null +++ b/evidence/native-sandbox/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-11-01 00:22:29, commit: unknown +# command: python scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/native-sandbox/configs.txt b/evidence/native-sandbox/configs.txt new file mode 100644 index 0000000..575837b --- /dev/null +++ b/evidence/native-sandbox/configs.txt @@ -0,0 +1,28 @@ +# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning +# date: 2026-11-01 00:22:31, commit: unknown +# command: python student-tests/check_configs.py +# exit code: 0, 0.7 s + +--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0 + +--- promtool check config (infra/prometheus/prometheus.yml): exit 0 + +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool check rules (recording + alerts): exit 0 +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml + SUCCESS: 6 rules found + +Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml + SUCCESS: 10 rules found +--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0 +SUCCESS +--- loki -verify-config (infra/loki/loki.yaml): exit 0 + +--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0 + +--- Grafana: not reachable at http://127.0.0.1:3000, skipped (local binaries mode) +PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid diff --git a/evidence/native-sandbox/dashboard.txt b/evidence/native-sandbox/dashboard.txt new file mode 100644 index 0000000..6003ba1 --- /dev/null +++ b/evidence/native-sandbox/dashboard.txt @@ -0,0 +1,36 @@ +# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels +# date: 2026-11-01 00:23:49, commit: unknown +# command: python student-tests/check_dashboard.py +# exit code: 0, 0.5 s + +PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident) + empty now: CPU контейнеров / лимит + empty now: Collector: потери и отказы + empty now: SDK приложений: экспорт и потери + empty now: alerts.yml: MessengerDbPoolSaturated + empty now: alerts.yml: MessengerHighErrorRate + empty now: alerts.yml: MessengerQueueLagGrowing + empty now: alerts.yml: MessengerWorkerNoConsumers + empty now: alerts.yml: TelemetryCollectorDown + empty now: alerts.yml: TelemetryExportFailing + empty now: Алерты Prometheus (pending/firing) + empty now: Память: RSS процесса и контейнер +LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error" +TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 5 trace(s) + +data: http_server_request_duration_seconds_count{job="messenger-api"} 39 series <- HTTP requests/duration (3) +data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3) +data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4) +data: messenger_queue_depth 2 series <- queue depth (4) +data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4) +data: db_client_connection_count 4 series <- DB pool used/idle (4) +data: db_client_connection_pending_requests 2 series <- DB pool waiting (4) +data: messenger_worker_busy 1 series <- worker busy (4) +data: messenger_worker_capacity 1 series <- worker capacity (4) +data: process_cpu_time_seconds_total 12 series <- process CPU (4) +data: process_memory_usage_bytes 6 series <- process RAM (4) +data: otelcol_exporter_queue_size 3 series <- collector queue (4) + +labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, error_type, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version +http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready'] +PASS: every dashboard/rule query runs, key signals have data, no forbidden labels diff --git a/evidence/native-sandbox/idempotency.txt b/evidence/native-sandbox/idempotency.txt new file mode 100644 index 0000000..c2efb08 --- /dev/null +++ b/evidence/native-sandbox/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-11-01 00:52:50, commit: unknown +# command: python student-tests/test_idempotency.py +# exit code: 0, 2.2 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'a1a1a1a1a1a1': 10, 'a2a2a2a2a2a2': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/native-sandbox/incident-collector-down.json b/evidence/native-sandbox/incident-collector-down.json new file mode 100644 index 0000000..2910ce1 --- /dev/null +++ b/evidence/native-sandbox/incident-collector-down.json @@ -0,0 +1,39 @@ +{ + "scenario": "collector-down", + "date": "2026-11-01 00:39:14", + "outage_s": 90, + "baseline": { + "requests": 1002, + "codes": { + "201": 1002 + }, + "success_ratio": 1.0, + "success_rps": 66.8, + "p95_ms_successful": 15.2 + }, + "during": { + "requests": 6263, + "codes": { + "201": 6263 + }, + "success_ratio": 1.0, + "success_rps": 67.4, + "p95_ms_successful": 14.3 + }, + "api_mib_before": 280.5, + "api_mib_max": 292.5, + "sdk_lost_spans_api": { + "processor_queue_full": 0, + "exporter_failed": 24576 + }, + "sdk_exported_ok_after_restart": 10524, + "after": { + "requests": 1703, + "codes": { + "201": 1703 + }, + "success_ratio": 1.0, + "success_rps": 68.1, + "p95_ms_successful": 13.8 + } +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-collector.txt b/evidence/native-sandbox/incident-collector.txt new file mode 100644 index 0000000..8a94682 --- /dev/null +++ b/evidence/native-sandbox/incident-collector.txt @@ -0,0 +1,22 @@ +# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted +# date: 2026-11-01 00:41:37, commit: unknown +# command: python student-tests/incidents.py collector-down +# exit code: 0, 143.3 s + +=== collector-down +[ 20.3s] baseline: {'requests': 1002, 'codes': {'201': 1002}, 'success_ratio': 1.0, 'success_rps': 66.8, 'p95_ms_successful': 15.2}, api containers 280.5 MiB +[ 20.6s] docker compose stop otel-collector +[ 20.8s] {'t': 0.3, 'api_mib': 283.0} +[ 31.0s] {'t': 10.5, 'api_mib': 283.2} +[ 41.3s] {'t': 20.8, 'api_mib': 288.6} +[ 51.5s] {'t': 31.0, 'api_mib': 289.0} +[ 61.7s] {'t': 41.2, 'api_mib': 289.7} +[ 71.9s] {'t': 51.4, 'api_mib': 292.5} +[ 82.1s] {'t': 61.6, 'api_mib': 292.4} +[ 92.4s] {'t': 71.9, 'api_mib': 292.3} +[ 102.6s] {'t': 82.1, 'api_mib': 291.2} +[ 113.2s] docker compose start otel-collector +[ 143.2s] API during collector outage: {'requests': 6263, 'codes': {'201': 6263}, 'success_ratio': 1.0, 'success_rps': 67.4, 'p95_ms_successful': 14.3}; api memory 280.5 -> max 292.5 MiB; SDK lost spans (api): {'processor_queue_full': 0, 'exporter_failed': 24576}; exported OK in the last minute after restart: 10524 +[ 143.2s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-collector-down.json +PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start +1 passed, 0 failed diff --git a/evidence/native-sandbox/incident-db-slow.json b/evidence/native-sandbox/incident-db-slow.json new file mode 100644 index 0000000..d73c0ac --- /dev/null +++ b/evidence/native-sandbox/incident-db-slow.json @@ -0,0 +1,82 @@ +{ + "scenario": "db-slow", + "date": "2026-11-01 00:42:49", + "lock_seconds": 210, + "baseline": { + "requests": 2648, + "codes": { + "201": 2648 + }, + "success_ratio": 1.0, + "success_rps": 132.4, + "p95_ms_successful": 18.5, + "server_p95_s": 0.0214 + }, + "firing_after_lock_s": 120.1, + "diagnosis": { + "http_5xx_ratio_1m": 1.0, + "http_p95_messages_s": 7.375, + "pool_used": 8.0, + "pool_wait_p95_s": 0.001, + "status_codes_1m": { + "201": 0.0, + "200": 0.0, + "404": 0.0, + "400": 0.0, + "409": 0.0, + "503": 1.6 + }, + "reads_during_lock_ms": { + "GET /users/me + GET /chats, avg per request": 4.7, + "max": 5.4 + }, + "api_log_db_unavailable_by_error_type": { + "QueryCanceled": 184 + }, + "error_trace": { + "trace_id": "1c7f8dd06427574a5f155558aa4cd975", + "server_ms": 5006.7, + "slowest_db_span": "INSERT", + "slowest_db_ms": 5000.6, + "db_span_status": "ERROR", + "db_share": 0.999, + "tree": [ + "- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503", + " - [api] redis GET (CLIENT, 0.2 ms)", + " - [api] SELECT (CLIENT, 0.4 ms)", + " - [api] INSERT (CLIENT, 5000.6 ms) ERROR" + ] + }, + "handler_cpu_contrast": { + "trace_id": "a6da2e3fe7b0c229773f36fa507293ca", + "server_ms": 34.0, + "argon2id_ms": 29.0, + "db_ms_total": 0.6, + "tree": [ + "- [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201", + " - [api] argon2id hash (INTERNAL, 29.0 ms)", + " - [api] INSERT (CLIENT, 0.6 ms)" + ] + } + }, + "during_lock": { + "requests": 326, + "codes": { + "503": 320, + "201": 6 + }, + "success_ratio": 0.0184, + "success_rps": 0.0, + "p95_ms_successful": 2718.4 + }, + "resolved_after_unlock_s": 30.0, + "after": { + "requests": 3298, + "codes": { + "201": 3298 + }, + "success_ratio": 1.0, + "success_rps": 131.9, + "p95_ms_successful": 21.0 + } +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-db.txt b/evidence/native-sandbox/incident-db.txt new file mode 100644 index 0000000..b0471c0 --- /dev/null +++ b/evidence/native-sandbox/incident-db.txt @@ -0,0 +1,56 @@ +# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved +# date: 2026-11-01 00:47:15, commit: unknown +# command: python student-tests/incidents.py db-slow +# exit code: 0, 265.4 s + +=== db-slow +[ 0.1s] load: 8 threads POST /messages +[ 25.1s] baseline: {'requests': 2648, 'codes': {'201': 2648}, 'success_ratio': 1.0, 'success_rps': 132.4, 'p95_ms_successful': 18.5, 'server_p95_s': 0.0214} +[ 25.1s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait) +[ 25.1s] 5xx ratio=0.0 p95=0.021 s pool used=0.0 waiting=0.0 +[ 30.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 35.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 40.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0 +[ 45.1s] 5xx ratio=0.005 p95=0.022 s pool used=8.0 waiting=0.0 +[ 50.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0 +[ 55.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0 +[ 60.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0 +[ 65.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0 +[ 70.1s] 5xx ratio=0.01 p95=5.688 s pool used=8.0 waiting=0.0 +[ 75.2s] 5xx ratio=0.069 p95=5.688 s pool used=8.0 waiting=0.0 +[ 80.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0 +[ 85.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0 +[ 90.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 95.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 100.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 105.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 110.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 115.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 120.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 125.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 130.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 135.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 140.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0 +[ 145.2s] ALERT MessengerHighErrorRate firing (120.1 s after lock) +[ 151.5s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 4.7, "max": 5.4}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 184}} +[ 151.5s] error trace (DB): {"trace_id": "1c7f8dd06427574a5f155558aa4cd975", "server_ms": 5006.7, "slowest_db_span": "INSERT", "slowest_db_ms": 5000.6, "db_span_status": "ERROR", "db_share": 0.999} + - [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.4 ms) + - [api] INSERT (CLIENT, 5000.6 ms) ERROR +[ 151.5s] contrast, slow handler (CPU): {"trace_id": "a6da2e3fe7b0c229773f36fa507293ca", "server_ms": 34.0, "argon2id_ms": 29.0, "db_ms_total": 0.6} + - [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201 + - [api] argon2id hash (INTERNAL, 29.0 ms) + - [api] INSERT (CLIENT, 0.6 ms) +[ 235.4s] lock released (psql exit 0), 210.3 s after LOCK +[ 235.4s] ... MessengerHighErrorRate: firing (0 s) +[ 240.4s] ... MessengerHighErrorRate: firing (5 s) +[ 245.4s] ... MessengerHighErrorRate: firing (10 s) +[ 250.4s] ... MessengerHighErrorRate: firing (15 s) +[ 255.4s] ... MessengerHighErrorRate: firing (20 s) +[ 260.4s] ... MessengerHighErrorRate: firing (25 s) +[ 265.4s] ALERT MessengerHighErrorRate resolved 30.0 s after unlock +[ 265.4s] load: during lock {'requests': 326, 'codes': {'503': 320, '201': 6}, 'success_ratio': 0.0184, 'success_rps': 0.0, 'p95_ms_successful': 2718.4}; after {'requests': 3298, 'codes': {'201': 3298}, 'success_ratio': 1.0, 'success_rps': 131.9, 'p95_ms_successful': 21.0} +[ 265.4s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-db-slow.json +PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved +1 passed, 0 failed diff --git a/evidence/native-sandbox/incident-telemetry-down.json b/evidence/native-sandbox/incident-telemetry-down.json new file mode 100644 index 0000000..9e2683d --- /dev/null +++ b/evidence/native-sandbox/incident-telemetry-down.json @@ -0,0 +1,307 @@ +{ + "scenario": "telemetry-down", + "date": "2026-11-01 00:32:59", + "stopped": [ + "tempo", + "loki" + ], + "outage_s": 150, + "baseline": { + "requests": 955, + "codes": { + "201": 955 + }, + "success_ratio": 1.0, + "success_rps": 63.7, + "p95_ms_successful": 17.9 + }, + "collector_heap_before_mib": 44.1, + "collector_container_before_mib": 224.4, + "during": { + "requests": 11500, + "codes": { + "201": 11500 + }, + "success_ratio": 1.0, + "success_rps": 68.3, + "p95_ms_successful": 13.2 + }, + "samples": [ + { + "t": 0.0, + "collector_heap_mib": 39.8, + "collector_rss_mib": 228.5, + "collector_container_mib": 228.6, + "queue": { + "otlp_http/loki": 0.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 6.0 + }, + "alert": "inactive" + }, + { + "t": 10.1, + "collector_heap_mib": 52.2, + "collector_rss_mib": 228.6, + "collector_container_mib": 228.6, + "queue": { + "otlp_http/loki": 5.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 10.0 + }, + "alert": "inactive" + }, + { + "t": 20.3, + "collector_heap_mib": 52.2, + "collector_rss_mib": 228.6, + "collector_container_mib": 228.7, + "queue": { + "otlp_http/loki": 5.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 10.0 + }, + "alert": "inactive" + }, + { + "t": 30.4, + "collector_heap_mib": 63.3, + "collector_rss_mib": 230.2, + "collector_container_mib": 233.6, + "queue": { + "otlp_http/loki": 12.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 14.0 + }, + "alert": "inactive" + }, + { + "t": 40.6, + "collector_heap_mib": 48.5, + "collector_rss_mib": 234.5, + "collector_container_mib": 234.5, + "queue": { + "otlp_http/loki": 20.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 18.0 + }, + "alert": "pending" + }, + { + "t": 50.7, + "collector_heap_mib": 48.5, + "collector_rss_mib": 234.5, + "collector_container_mib": 234.6, + "queue": { + "otlp_http/loki": 20.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 18.0 + }, + "alert": "pending" + }, + { + "t": 60.8, + "collector_heap_mib": 59.5, + "collector_rss_mib": 234.6, + "collector_container_mib": 234.9, + "queue": { + "otlp_http/loki": 27.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 22.0 + }, + "alert": "pending" + }, + { + "t": 71.0, + "collector_heap_mib": 70.7, + "collector_rss_mib": 239.4, + "collector_container_mib": 242.2, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 26.0 + }, + "alert": "pending" + }, + { + "t": 81.2, + "collector_heap_mib": 70.7, + "collector_rss_mib": 239.4, + "collector_container_mib": 247.4, + "queue": { + "otlp_http/loki": 35.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 26.0 + }, + "alert": "pending" + }, + { + "t": 91.3, + "collector_heap_mib": 58.1, + "collector_rss_mib": 247.4, + "collector_container_mib": 247.4, + "queue": { + "otlp_http/loki": 42.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 30.0 + }, + "alert": "pending" + }, + { + "t": 101.4, + "collector_heap_mib": 69.3, + "collector_rss_mib": 247.5, + "collector_container_mib": 247.5, + "queue": { + "otlp_http/loki": 50.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 32.0 + }, + "alert": "firing" + }, + { + "t": 111.6, + "collector_heap_mib": 69.3, + "collector_rss_mib": 247.5, + "collector_container_mib": 249.7, + "queue": { + "otlp_http/loki": 50.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 32.0 + }, + "alert": "firing" + }, + { + "t": 121.8, + "collector_heap_mib": 80.2, + "collector_rss_mib": 249.9, + "collector_container_mib": 258.8, + "queue": { + "otlp_http/loki": 57.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 36.0 + }, + "alert": "firing" + }, + { + "t": 131.9, + "collector_heap_mib": 92.7, + "collector_rss_mib": 262.3, + "collector_container_mib": 265.4, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 40.0 + }, + "alert": "firing" + }, + { + "t": 142.1, + "collector_heap_mib": 92.7, + "collector_rss_mib": 262.3, + "collector_container_mib": 273.3, + "queue": { + "otlp_http/loki": 63.0, + "otlp_http/prometheus": 0.0, + "otlp_grpc/tempo": 40.0 + }, + "alert": "firing" + } + ], + "collector_heap_max_mib": 92.7, + "collector_rss_max_mib": 262.3, + "collector_container_max_mib": 273.3, + "TelemetryExportFailing_firing_at_s": 101.4, + "queue_drained_after_start_s": 80.0, + "probe_traces": [ + { + "sent_at_s": 0.0, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 10.1, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 20.3, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 30.4, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 40.6, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 50.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 60.8, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 71.0, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 81.2, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 91.3, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 101.4, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 111.6, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 121.7, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 131.9, + "http": 201, + "trace_in_tempo": true + }, + { + "sent_at_s": 142.1, + "http": 201, + "trace_in_tempo": false + } + ], + "lost_in_collector": { + "spans": 4096, + "log_records": 269 + }, + "resolved_after_start_s": 155.4, + "after": { + "requests": 10186, + "codes": { + "201": 10186 + }, + "success_ratio": 1.0, + "success_rps": 67.7, + "p95_ms_successful": 14.2 + } +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-telemetry.txt b/evidence/native-sandbox/incident-telemetry.txt new file mode 100644 index 0000000..8add4c3 --- /dev/null +++ b/evidence/native-sandbox/incident-telemetry.txt @@ -0,0 +1,41 @@ +# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted +# date: 2026-11-01 00:39:14, commit: unknown +# command: python student-tests/incidents.py telemetry-down +# exit code: 0, 374.8 s + +=== telemetry-down +[ 20.2s] baseline: {'requests': 955, 'codes': {'201': 955}, 'success_ratio': 1.0, 'success_rps': 63.7, 'p95_ms_successful': 17.9}, collector heap 44.1 MiB, container 224.4 MiB +[ 50.9s] docker compose stop tempo loki: stopped +[ 51.1s] {'t': 0.0, 'collector_heap_mib': 39.8, 'collector_rss_mib': 228.5, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 0.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 6.0}, 'alert': 'inactive'} +[ 61.2s] {'t': 10.1, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'} +[ 71.3s] {'t': 20.3, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.7, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'} +[ 81.5s] {'t': 30.4, 'collector_heap_mib': 63.3, 'collector_rss_mib': 230.2, 'collector_container_mib': 233.6, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 14.0}, 'alert': 'inactive'} +[ 91.6s] {'t': 40.6, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.5, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'} +[ 101.8s] {'t': 50.7, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.6, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'} +[ 112.0s] {'t': 60.8, 'collector_heap_mib': 59.5, 'collector_rss_mib': 234.6, 'collector_container_mib': 234.9, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'pending'} +[ 122.1s] {'t': 71.0, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 242.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'} +[ 132.2s] {'t': 81.2, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'} +[ 142.4s] {'t': 91.3, 'collector_heap_mib': 58.1, 'collector_rss_mib': 247.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 30.0}, 'alert': 'pending'} +[ 152.5s] {'t': 101.4, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 247.5, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'} +[ 162.7s] {'t': 111.6, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 249.7, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'} +[ 172.9s] {'t': 121.8, 'collector_heap_mib': 80.2, 'collector_rss_mib': 249.9, 'collector_container_mib': 258.8, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 36.0}, 'alert': 'firing'} +[ 183.0s] {'t': 131.9, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 265.4, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'} +[ 193.2s] {'t': 142.1, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 273.3, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'} +[ 219.3s] docker compose start tempo loki +[ 299.3s] exporter queues drained 80.0 s after start +[ 324.7s] ... TelemetryExportFailing: firing (0 s) +[ 329.7s] ... TelemetryExportFailing: firing (5 s) +[ 334.7s] ... TelemetryExportFailing: firing (10 s) +[ 339.7s] ... TelemetryExportFailing: firing (15 s) +[ 344.7s] ... TelemetryExportFailing: firing (20 s) +[ 349.7s] ... TelemetryExportFailing: firing (25 s) +[ 354.7s] ... TelemetryExportFailing: firing (30 s) +[ 359.7s] ... TelemetryExportFailing: firing (35 s) +[ 364.7s] ... TelemetryExportFailing: firing (40 s) +[ 369.7s] ... TelemetryExportFailing: firing (45 s) +[ 374.7s] API during outage: {'requests': 11500, 'codes': {'201': 11500}, 'success_ratio': 1.0, 'success_rps': 68.3, 'p95_ms_successful': 13.2} +[ 374.7s] probe traces: 14/15 reached Tempo (lost: sent at [142.1] s) +[ 374.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 269}; collector heap 44.1 -> max 92.7 MiB (memory_limiter 200 MiB), container max 273.3 MiB +[ 374.7s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-telemetry-down.json +PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved +1 passed, 0 failed diff --git a/evidence/native-sandbox/incident-worker-stop.json b/evidence/native-sandbox/incident-worker-stop.json new file mode 100644 index 0000000..fe3a2fa --- /dev/null +++ b/evidence/native-sandbox/incident-worker-stop.json @@ -0,0 +1,49 @@ +{ + "scenario": "worker-stop", + "date": "2026-11-01 00:26:35", + "firing_after_stop_s": { + "MessengerWorkerNoConsumers": 66.4, + "MessengerQueueLagGrowing": 111.4 + }, + "diagnosis": { + "queue_consumers": 0.0, + "broker_queue_depth": 3.0, + "jobs_backlog_queued": 3.0, + "oldest_job_age_s": 97.3, + "outbox_oldest_age_s": 0.0, + "published_ok_last_5m": 3.102218085931441, + "worker_metrics_last_seen_s": 110.0, + "last_worker_log": { + "event": "service.stopped", + "message": "stopped" + }, + "upload_trace_span_names": [ + "INSERT", + "NOTIFY", + "POST /api/v1/chats/{chat_id}/attachments", + "S3 PutObject", + "SELECT", + "UPDATE", + "create attachments", + "inspect image", + "publish attachments", + "redis GET" + ], + "upload_trace_has_consumer_span": false + }, + "ready_after_start_s": [ + 1.5, + 1.6, + 2.0 + ], + "resolved_after_start_s": { + "MessengerWorkerNoConsumers": 17.0, + "MessengerQueueLagGrowing": 17.0 + }, + "after_recovery": { + "jobs_backlog_queued": 0.0, + "oldest_job_age_s": 0.0, + "queue_consumers": 2.0 + }, + "time_to_ready_p95_5m_s": 117.0 +} \ No newline at end of file diff --git a/evidence/native-sandbox/incident-worker.txt b/evidence/native-sandbox/incident-worker.txt new file mode 100644 index 0000000..88166d8 --- /dev/null +++ b/evidence/native-sandbox/incident-worker.txt @@ -0,0 +1,43 @@ +# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved +# date: 2026-11-01 00:28:44, commit: unknown +# command: python student-tests/incidents.py worker-stop +# exit code: 0, 128.6 s + +=== worker-stop +[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s +[ 1.3s] docker compose stop worker +[ 1.4s] uploaded 3 attachments: 202 queued (API accepts without worker) +[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {} +[ 6.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 16.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 21.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 26.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 31.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 36.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 41.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 46.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'} +[ 51.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 56.4s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 61.5s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'} +[ 66.5s] ALERT MessengerWorkerNoConsumers firing (66.4 s after stop) +[ 66.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 71.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 76.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 81.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 86.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 91.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 96.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 101.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 106.5s] consumers=0.0 backlog queued=3.0 oldest age=97.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'} +[ 111.5s] ALERT MessengerQueueLagGrowing firing (111.4 s after stop) +[ 111.5s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 3.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 97.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 3.102218085931441, "worker_metrics_last_seen_s": 110.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET"], "upload_trace_has_consumer_span": false} +[ 111.6s] docker compose start worker +[ 113.5s] all attachments ready 2.0 s after worker start; backlog now 3.0 +[ 113.5s] ... MessengerWorkerNoConsumers: firing (0 s) +[ 118.5s] ... MessengerWorkerNoConsumers: firing (5 s) +[ 123.5s] ... MessengerWorkerNoConsumers: firing (10 s) +[ 128.5s] alerts resolved: {'MessengerWorkerNoConsumers': 17.0, 'MessengerQueueLagGrowing': 17.0}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0} +[ 128.5s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-worker-stop.json +PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-concurrency.txt b/evidence/native-sandbox/jobs-concurrency.txt new file mode 100644 index 0000000..f5f5735 --- /dev/null +++ b/evidence/native-sandbox/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-11-01 00:52:34, commit: unknown +# command: python student-tests/test_jobs.py concurrency +# exit code: 0, 7.1 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 4.1 s after worker start, peak processing 2 of 8 (22 samples) +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-outbox.txt b/evidence/native-sandbox/jobs-outbox.txt new file mode 100644 index 0000000..68b16d5 --- /dev/null +++ b/evidence/native-sandbox/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-11-01 00:51:38, commit: unknown +# command: python student-tests/test_jobs.py outbox-crash +# exit code: 0, 7.4 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 18ab06f7-657b-4508-9ba5-844bd2c7ba2e ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-recovery.txt b/evidence/native-sandbox/jobs-recovery.txt new file mode 100644 index 0000000..41b8503 --- /dev/null +++ b/evidence/native-sandbox/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-11-01 00:48:13, commit: unknown +# command: python student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 44.2 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment 748b0b56-e355-48a0-9ac6-4b731a3432a9 +message with attachment 201, content 409, other messages 201; job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job 5784e1e7-7173-4a27-a3ad-72216f4a43a9 +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-retry.txt b/evidence/native-sandbox/jobs-retry.txt new file mode 100644 index 0000000..83f800a --- /dev/null +++ b/evidence/native-sandbox/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-11-01 00:52:27, commit: unknown +# command: python student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 49.1 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.40 s and 3.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id fce6904a-fe4b-429c-821a-e3b55a43b0ef attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] +4 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-variants.txt b/evidence/native-sandbox/jobs-variants.txt new file mode 100644 index 0000000..691301a --- /dev/null +++ b/evidence/native-sandbox/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-11-01 00:47:29, commit: unknown +# command: python student-tests/test_jobs.py variants +# exit code: 0, 14.3 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/native-sandbox/load.json b/evidence/native-sandbox/load.json new file mode 100644 index 0000000..48e138d --- /dev/null +++ b/evidence/native-sandbox/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.083, + "concurrency": 4, + "successful_requests": 8762, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 291.265, + "successful_latency_ms": { + "p50": 11.557, + "p95": 26.659, + "p99": 56.425 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/load.txt b/evidence/native-sandbox/load.txt new file mode 100644 index 0000000..5ecdade --- /dev/null +++ b/evidence/native-sandbox/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-11-01 00:42:08, commit: unknown +# command: python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.4 s + +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.083, + "concurrency": 4, + "successful_requests": 8762, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 291.265, + "successful_latency_ms": { + "p50": 11.557, + "p95": 26.659, + "p99": 56.425 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/race.txt b/evidence/native-sandbox/race.txt new file mode 100644 index 0000000..587dc51 --- /dev/null +++ b/evidence/native-sandbox/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-11-01 00:52:53, commit: unknown +# command: python student-tests/test_race.py +# exit code: 0, 1.5 s + +sent 100 messages from 10 clients in 0.34 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/native-sandbox/smoke.txt b/evidence/native-sandbox/smoke.txt new file mode 100644 index 0000000..dd1fccd --- /dev/null +++ b/evidence/native-sandbox/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment +# date: 2026-11-01 00:22:30, commit: unknown +# command: python tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 1.7 s + +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/native-sandbox/student-tests.txt b/evidence/native-sandbox/student-tests.txt new file mode 100644 index 0000000..a356d19 --- /dev/null +++ b/evidence/native-sandbox/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-11-01 00:52:52, commit: unknown +# command: python student-tests/test_api.py +# exit code: 0, 2.0 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/native-sandbox/summary.md b/evidence/native-sandbox/summary.md new file mode 100644 index 0000000..e53e6a2 --- /dev/null +++ b/evidence/native-sandbox/summary.md @@ -0,0 +1,48 @@ +# Сводка проверок (нативная песочница, без Docker) + +Стенд теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, dispatcher, worker, janitor (Python 3.11, requirements.txt), otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из infra/ (адреса -> 127.0.0.1). Команды `docker compose` тестов выполнялись через подмену на управление процессами. Машина: Linux x86_64, 2 vCPU, 8 GiB RAM. + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt | +| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 2 | evidence/native-sandbox/smoke.txt | +| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 1 | evidence/native-sandbox/configs.txt | +| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 77 | evidence/native-sandbox/telemetry.txt | +| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 0 | evidence/native-sandbox/dashboard.txt | +| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 52 | evidence/native-sandbox/telemetry-history.txt | +| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 129 | evidence/native-sandbox/incident-worker.txt | +| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 265 | evidence/native-sandbox/incident-db.txt | +| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 375 | evidence/native-sandbox/incident-telemetry.txt | +| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 143 | evidence/native-sandbox/incident-collector.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 14 | evidence/native-sandbox/jobs-variants.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 44 | evidence/native-sandbox/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 7 | evidence/native-sandbox/jobs-outbox.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 14 | evidence/native-sandbox/attachments.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/native-sandbox/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 2 | evidence/native-sandbox/student-tests.txt | +| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/native-sandbox/load.txt | + +# Замеры + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с | +| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) | + +Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной). diff --git a/evidence/native-sandbox/telemetry-history.txt b/evidence/native-sandbox/telemetry-history.txt new file mode 100644 index 0000000..381a337 --- /dev/null +++ b/evidence/native-sandbox/telemetry-history.txt @@ -0,0 +1,9 @@ +# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) +# date: 2026-11-01 00:26:35, commit: unknown +# command: python student-tests/test_telemetry.py history +# exit code: 0, 52.2 s + +before restart: trace 52668075eebc7fd4938de9690ba2aad1 found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 719 +after restart of prometheus, loki, tempo: trace 52668075eebc7fd4938de9690ba2aad1 (4 spans), 1 log record(s), metric at t-30s = 719 (same value) +PASS: persistent volumes keep traces, logs and metrics across restarts of the stores +1 passed, 0 failed diff --git a/evidence/native-sandbox/telemetry.txt b/evidence/native-sandbox/telemetry.txt new file mode 100644 index 0000000..bcc0d51 --- /dev/null +++ b/evidence/native-sandbox/telemetry.txt @@ -0,0 +1,145 @@ +# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets +# date: 2026-11-01 00:23:48, commit: unknown +# command: python student-tests/test_telemetry.py request upload-chain retry-chain replay-link +# exit code: 0, 77.1 s + +POST message: 201, X-Request-Id=4d281ed1-39de-484a-add2-8480ef33ab9c, trace_id=97ebaba763fec926fa1e1ed3fc0fae7a (client traceparent) +- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5.8 ms) http.response.status_code=201 + - [api] redis GET (CLIENT, 0.3 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] INSERT (CLIENT, 0.8 ms) +Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (5.9 ms)' event=http.request severity=INFO service=messenger-api trace_id=97ebaba763fec926fa1e1ed3fc0fae7a span_id=82ab66080cb50d2a request_id=4d281ed1-39de-484a-add2-8480ef33ab9c +PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 64.6 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.4 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] inspect image (INTERNAL, 45.7 ms) + - [api] INSERT (CLIENT, 1.1 ms) + - [api] S3 PutObject (CLIENT, 5.1 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1 + - [api] UPDATE (CLIENT, 0.9 ms) + - [api] create attachments (PRODUCER, 3.5 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.9 ms) + - [api] INSERT (CLIENT, 0.9 ms) + - [api] INSERT (CLIENT, 0.8 ms) + - [api] NOTIFY (CLIENT, 0.3 ms) + - [dispatcher] publish attachments (PRODUCER, 4.4 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.9 ms) + - [worker] process attachments.process.v1 (CONSUMER, 280.0 ms) messenger.job.attempt=1 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 1.4 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 1.4 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] S3 GetObject (CLIENT, 2.9 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1 + - [worker] verify source sha256 (INTERNAL, 0.1 ms) + - [worker] imaging (INTERNAL, 238.2 ms) + - [worker] S3 PutObject (CLIENT, 4.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/original + - [worker] S3 PutObject (CLIENT, 3.4 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/crop.jpg + - [worker] commit result (INTERNAL, 7.5 ms) + - [worker] SELECT (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 1.0 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] UPDATE (CLIENT, 1.0 ms) + - [worker] UPDATE (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] cleanup staging (INTERNAL, 4.1 ms) + - [worker] S3 DeleteObject (CLIENT, 2.4 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1 + - [worker] UPDATE (CLIENT, 1.4 ms) +worker logs with trace_id 0eecc096d0191a19e41ad1547110183c: ['job.claimed', 'job.ready'] +PASS: one trace 0eecc096d0191a19e41ad1547110183c: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker) +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 17.4 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.3 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] inspect image (INTERNAL, 0.3 ms) + - [api] INSERT (CLIENT, 0.7 ms) + - [api] S3 PutObject (CLIENT, 5.0 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [api] UPDATE (CLIENT, 0.6 ms) + - [api] create attachments (PRODUCER, 2.9 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.6 ms) + - [api] INSERT (CLIENT, 0.5 ms) + - [api] INSERT (CLIENT, 0.5 ms) + - [api] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 3.2 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.5 ms) + - [worker] process attachments.process.v1 (CONSUMER, 255.7 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable + - [worker] SELECT (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] S3 GetObject (CLIENT, 3.4 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 238.9 ms) + - [worker] SELECT (CLIENT, 0.6 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] create attachments (PRODUCER, 1.5 ms) messenger.job.attempt=2 + - [worker] INSERT (CLIENT, 0.8 ms) + - [worker] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 3.3 ms) messenger.job.attempt=2 + - [dispatcher] UPDATE (CLIENT, 0.6 ms) + - [worker] process attachments.process.v1 (CONSUMER, 284.4 ms) messenger.job.attempt=2 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.7 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] S3 GetObject (CLIENT, 3.5 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [worker] verify source sha256 (INTERNAL, 0.1 ms) + - [worker] imaging (INTERNAL, 250.7 ms) + - [worker] S3 PutObject (CLIENT, 4.7 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/original + - [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 3.0 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/crop.jpg + - [worker] commit result (INTERNAL, 6.6 ms) + - [worker] SELECT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.8 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.5 ms) + - [worker] cleanup staging (INTERNAL, 4.6 ms) + - [worker] S3 DeleteObject (CLIENT, 2.7 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df + - [worker] UPDATE (CLIENT, 1.3 ms) + - [worker] INSERT (CLIENT, 0.5 ms) +worker log: 'job 9a82f341-ec57-49e5-aa86-6263681529c1 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable +messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1 +PASS: retry in one trace 1abe27ebd519ac0854a844d18d679bf6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged +original trace 5b3ea90278cb5e18b6ad08cd32349c0e: 5 attempts, last outcome failed +- [ops] ops replay (INTERNAL, 16.5 ms) + - [ops] SELECT (CLIENT, 2.3 ms) + - [ops] UPDATE (CLIENT, 0.9 ms) + - [ops] UPDATE (CLIENT, 0.8 ms) + - [ops] INSERT (CLIENT, 0.7 ms) + - [ops] create attachments (PRODUCER, 1.1 ms) links=1 messenger.job.attempt=6 + - [ops] INSERT (CLIENT, 0.7 ms) + - [ops] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 5.3 ms) messenger.job.attempt=6 + - [worker] process attachments.process.v1 (CONSUMER, 281.7 ms) messenger.job.attempt=6 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] S3 GetObject (CLIENT, 2.7 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00 + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 251.3 ms) + - [worker] S3 PutObject (CLIENT, 3.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/original + - [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/crop.jpg + - [worker] commit result (INTERNAL, 6.0 ms) + - [worker] SELECT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.7 ms) + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] UPDATE (CLIENT, 0.7 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] cleanup staging (INTERNAL, 5.9 ms) + - [worker] S3 DeleteObject (CLIENT, 3.3 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00 + - [worker] UPDATE (CLIENT, 2.1 ms) + - [dispatcher] UPDATE (CLIENT, 1.3 ms) +replay trace 6d0327f64ceb5758c4a19b30aceed617: create span links -> ['5b3ea90278cb5e18b6ad08cd32349c0e'] (original upload trace 5b3ea90278cb5e18b6ad08cd32349c0e) +PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it +4 passed, 0 failed diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..61f5154 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,40 @@ +# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) +# date: 2026-11-01 20:46:08, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 62.0 s + +before: container, ID, started at + messenger-lab6-api-1 171a473b0d28 2026-11-01T17:43:16.63584434Z + messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:43:36.466962173Z + messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:31:14.251298706Z + messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:06:55.450862118Z + messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:07:17.636225869Z + messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:07:20.27137747Z + messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:22:20.690589644Z + messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:27:23.730596273Z + messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:04.263026279Z + messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:09:18.447414724Z + messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:30:47.892983401Z + messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:44:43.823155102Z + messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:40:41.919464505Z + messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:22:20.681510324Z + messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:38:33.102546081Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 60.9 s): container, ID, started at, restarted? + messenger-lab6-api-1 171a473b0d28 2026-11-01T17:46:02.488153329Z same container: True, restarted: True + messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:46:02.317979748Z same container: True, restarted: True + messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:46:00.495060123Z same container: True, restarted: True + messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:45:57.449417453Z same container: True, restarted: True + messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:45:57.621731679Z same container: True, restarted: True + messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:46:05.984667885Z same container: True, restarted: True + messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:45:21.206788727Z same container: True, restarted: True + messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:45:56.829306342Z same container: True, restarted: True + messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:19.869545292Z same container: True, restarted: True + messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:45:20.880337926Z same container: True, restarted: True + messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:45:21.007029551Z same container: True, restarted: True + messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:45:19.748047785Z same container: True, restarted: True + messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:45:29.752000764Z same container: True, restarted: True + messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:45:50.068926735Z same container: True, restarted: True + messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:46:00.664535867Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, s3, rabbitmq, api, janitor, dispatcher, worker, lb, otel-collector, prometheus, loki, tempo, grafana (all api replicas; same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 0000000..fb910cf --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-11-01 20:42:41, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.2 s + +sent 100 messages from 10 clients in 0.30 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 0000000..c772f3a --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-11-01 20:42:56, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 3.2 s + +session key: session:0fe0e5550e07d95d...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 147 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 0000000..2f38cc2 --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,53 @@ +# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap +# date: 2026-11-01 20:38:59, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_s3_security.py +# exit code: 0, 10.5 s + +anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous head_object directly to S3 (from the data network) -> 403 +anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied +anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied +the same file through the API for a chat member -> 200 +PASS: the bucket is private: no anonymous read, list, write or policy read +API key: list buckets -> ['messenger-attachments'] +API key: get own object -> allowed +API key: put object -> allowed +API key: delete object -> allowed +API key: list bucket -> allowed +API key: create another bucket -> denied: AccessDenied +API key: make bucket public (put policy) -> denied: NotImplemented +API key: read bucket policy -> denied: NoSuchBucketPolicy +API key: delete bucket -> denied: AccessDenied +API key: change versioning -> denied: NotImplemented +PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied +compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand) +127.0.0.1:8333 from the host -> connection refused +127.0.0.1:9333 from the host -> connection refused +127.0.0.1:8888 from the host -> connection refused +127.0.0.1:19333 from the host -> connection refused +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of grafana: ['grafana_admin_password'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of loki: - +secrets of migrate: ['database_url'] +secrets of otel-collector: - +secrets of postgres: ['postgres_password'] +secrets of prometheus: - +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of telemetry-init: - +secrets of tempo: - +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init +s3-init: s3-init: bucket messenger-attachments exists +s3-init: s3-init: api key can put/get/list/delete objects of its bucket +s3-init: s3-init: api key cannot create buckets (AccessDenied) +s3-init: s3-init: bucket is private (anonymous list/get/put -> [403]) +after the repeated bootstrap: old file downloads, new upload works +PASS: bucket/user/policy bootstrap can be repeated safely +4 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..93d9450 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment +# date: 2026-11-01 20:07:47, commit: 470d9d6 + uncommitted changes +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 1.0 s + +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 0000000..f3f5ab4 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,66 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services +# date: 2026-11-01 20:07:46, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/check_stack.py +# exit code: 0, 3.1 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api 171a473b0d28 Up 26 seconds (healthy) 8080/tcp +api 1db2fedff78f Up 26 seconds (healthy) 8080/tcp +dispatcher 2f899202c4d1 Up 47 seconds (healthy) +grafana 23e7b8bb1b3a Up 49 seconds (healthy) 127.0.0.1:3000->3000/tcp +janitor 6dad6b402ed9 Up 26 seconds (healthy) +lb 0bffe8d8d7d2 Up 24 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +loki c62735c8baca Up 54 seconds 127.0.0.1:3100->3100/tcp +migrate dd9753ec4186 Exited (0) 48 seconds ago +otel-collector 8c416bdc0221 Up 55 seconds +postgres ced386ea9bbd Up 55 seconds (healthy) +prometheus 451bf5466e96 Up 55 seconds (healthy) 127.0.0.1:9090->9090/tcp +rabbitmq ec502659b532 Up 55 seconds (healthy) +redis 3c003ba9b32a Up 55 seconds (healthy) +s3 1e70a509e403 Up 55 seconds (healthy) +s3-init 64471229b155 Exited (0) 27 seconds ago +telemetry-init 809e227f3528 Exited (0) 55 seconds ago +tempo 3daf3848457f Up 54 seconds 127.0.0.1:3200->3200/tcp +worker cbd8138493bf Up 26 seconds (healthy) + +container image cpus memory host ports networks +messenger-lab6-api-1 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry +messenger-lab6-api-2 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry +messenger-lab6-dispatcher-1 messenger-lab6-dispatcher 0.25 128 MiB - data,telemetry +messenger-lab6-grafana-1 grafana/grafana:12.4.14 0.50 384 MiB 127.0.0.1:3000 ops,telemetry +messenger-lab6-janitor-1 messenger-lab6-janitor 0.25 192 MiB - data,telemetry +messenger-lab6-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab6-loki-1 grafana/loki:3.7.9 1.00 512 MiB 127.0.0.1:3100 ops,telemetry +messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.164.0 0.50 384 MiB - telemetry +messenger-lab6-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab6-prometheus-1 prom/prometheus:v3.15.1 1.00 512 MiB 127.0.0.1:9090 ops,telemetry +messenger-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data +messenger-lab6-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data +messenger-lab6-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data +messenger-lab6-tempo-1 grafana/tempo:3.1.1 1.00 512 MiB 127.0.0.1:3200 ops,telemetry +messenger-lab6-worker-1 messenger-lab6-worker 2.00 1536 MiB - data,telemetry + +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of grafana: ['grafana_admin_password'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of loki: - +secrets of migrate: ['database_url'] +secrets of otel-collector: - +secrets of postgres: ['postgres_password'] +secrets of prometheus: - +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of telemetry-init: - +secrets of tempo: - +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['171a473b0d28', '1db2fedff78f'] +/health/ready -> 200 from 1db2fedff78f +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/rabbitmq only in the internal data network, worker/dispatcher/janitor in data+telemetry, collector only in the internal telemetry network, prometheus/loki/tempo/grafana on 127.0.0.1 with persistent volumes, pinned images, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000..67775d3 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-11-01 20:42:40, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.8 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 0000000..72d4c17 --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,108 @@ +# Сводка проверок (Docker-стенд, 2026-11-01) + +Шаги запускались группами (`collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history`, затем каждый инцидент и `load` отдельно, затем `--from jobs-variants`); `load` выполнен дважды — в группе lab6 и в регрессии, evidence/load.txt и load.json — от второго прогона. + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/check.txt | +| stack | stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services | PASS | 3 | evidence/stack.txt | +| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 1 | evidence/smoke.txt | +| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 6 | evidence/configs.txt | +| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 79 | evidence/telemetry.txt | +| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 1 | evidence/dashboard.txt | +| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 39 | evidence/telemetry-history.txt | +| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 162 | evidence/incident-worker.txt | +| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 286 | evidence/incident-db.txt | +| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 380 | evidence/incident-telemetry.txt | +| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 156 | evidence/incident-collector.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt | +| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 73 | evidence/jobs-limits.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 17 | evidence/jobs-outbox.txt | +| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 157 | evidence/jobs-crash.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 78 | evidence/jobs-retry.txt | +| jobs-janitor-replay | lab5: janitor does not delete objects re-written by replay (review fix) | PASS | 187 | evidence/jobs-janitor-replay.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 16 | evidence/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt | +| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt | +| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 160 | evidence/consistency.txt | +| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 53 | evidence/upload-memory.txt | +| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt | +| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt | +| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 12 | evidence/db.txt | +| redis | Redis as the session store | PASS | 3 | evidence/redis.txt | +| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt | +| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 42 | evidence/failover.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| failures | Redis and PostgreSQL outages via the balancer | PASS | 56 | evidence/failures.txt | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 62 | evidence/persistence.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB. + +Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts): + +| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения | +| --- | --- | --- | --- | --- | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) | +| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) | +| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) | +| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) | + +Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0. + +Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок. + +| Отказ телеметрии | API во время отказа | Память | Потери телеметрии | +| --- | --- | --- | --- | +| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с | +| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) | + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 | +| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 | +| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 | +| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 | +| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | diff --git a/evidence/telemetry-history.txt b/evidence/telemetry-history.txt new file mode 100644 index 0000000..61abba2 --- /dev/null +++ b/evidence/telemetry-history.txt @@ -0,0 +1,9 @@ +# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) +# date: 2026-11-01 20:09:51, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_telemetry.py history +# exit code: 0, 38.6 s + +before restart: trace 0b9106c1057e9753186aec51b529946b found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 434 +after restart of prometheus, loki, tempo: trace 0b9106c1057e9753186aec51b529946b (4 spans), 1 log record(s), metric at t-30s = 434 (same value) +PASS: persistent volumes keep traces, logs and metrics across restarts of the stores +1 passed, 0 failed diff --git a/evidence/telemetry.txt b/evidence/telemetry.txt new file mode 100644 index 0000000..21e13af --- /dev/null +++ b/evidence/telemetry.txt @@ -0,0 +1,145 @@ +# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets +# date: 2026-11-01 20:09:11, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/test_telemetry.py request upload-chain retry-chain replay-link +# exit code: 0, 78.6 s + +POST message: 201, X-Request-Id=df174aef-1d0d-4340-b2d7-907241684f5e, trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce (client traceparent) +- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 4.6 ms) http.response.status_code=201 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.3 ms) + - [api] INSERT (CLIENT, 0.4 ms) +Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (4.6 ms)' event=http.request severity=INFO service=messenger-api trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce span_id=78cd8621fce4f3b8 request_id=df174aef-1d0d-4340-b2d7-907241684f5e +PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 13.5 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.4 ms) + - [api] inspect image (INTERNAL, 0.5 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238 + - [api] UPDATE (CLIENT, 0.7 ms) + - [api] create attachments (PRODUCER, 2.4 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.7 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] NOTIFY (CLIENT, 0.1 ms) + - [dispatcher] publish attachments (PRODUCER, 4.0 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.4 ms) + - [worker] process attachments.process.v1 (CONSUMER, 161.4 ms) messenger.job.attempt=1 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] S3 GetObject (CLIENT, 2.0 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238 + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 139.8 ms) + - [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/original + - [worker] S3 PutObject (CLIENT, 1.9 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 2.3 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/crop.jpg + - [worker] commit result (INTERNAL, 4.9 ms) + - [worker] SELECT (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] cleanup staging (INTERNAL, 4.7 ms) + - [worker] S3 DeleteObject (CLIENT, 2.2 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238 + - [worker] UPDATE (CLIENT, 2.2 ms) +worker logs with trace_id f5d1897588244cad5fc6ded83b930725: ['job.claimed', 'job.ready'] +PASS: one trace f5d1897588244cad5fc6ded83b930725: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker) +- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 12.6 ms) http.response.status_code=202 + - [api] redis GET (CLIENT, 0.2 ms) + - [api] SELECT (CLIENT, 0.5 ms) + - [api] inspect image (INTERNAL, 0.3 ms) + - [api] INSERT (CLIENT, 0.6 ms) + - [api] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [api] UPDATE (CLIENT, 0.6 ms) + - [api] create attachments (PRODUCER, 1.4 ms) messenger.job.attempt=1 + - [api] INSERT (CLIENT, 0.4 ms) + - [api] INSERT (CLIENT, 0.3 ms) + - [api] INSERT (CLIENT, 0.3 ms) + - [api] NOTIFY (CLIENT, 0.1 ms) + - [dispatcher] publish attachments (PRODUCER, 4.2 ms) messenger.job.attempt=1 + - [dispatcher] UPDATE (CLIENT, 0.4 ms) + - [worker] process attachments.process.v1 (CONSUMER, 135.6 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] S3 GetObject (CLIENT, 2.6 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 123.0 ms) + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] create attachments (PRODUCER, 1.0 ms) messenger.job.attempt=2 + - [worker] INSERT (CLIENT, 0.6 ms) + - [worker] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 5.2 ms) messenger.job.attempt=2 + - [dispatcher] UPDATE (CLIENT, 0.5 ms) + - [worker] process attachments.process.v1 (CONSUMER, 144.9 ms) messenger.job.attempt=2 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 125.7 ms) + - [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/original + - [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 1.7 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/crop.jpg + - [worker] commit result (INTERNAL, 4.4 ms) + - [worker] SELECT (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.4 ms) + - [worker] cleanup staging (INTERNAL, 4.5 ms) + - [worker] S3 DeleteObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c + - [worker] UPDATE (CLIENT, 2.5 ms) + - [worker] INSERT (CLIENT, 0.4 ms) +worker log: 'job 7bd96513-9f85-471e-b553-ec49f9fe4560 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable +messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1 +PASS: retry in one trace 5d34bb1eecbc90702d2f5e7de5b9bca6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged +original trace 38319800cdce5da47cb2b821dc03bf6e: 5 attempts, last outcome failed +- [ops] ops replay (INTERNAL, 12.3 ms) + - [ops] SELECT (CLIENT, 1.9 ms) + - [ops] UPDATE (CLIENT, 0.6 ms) + - [ops] UPDATE (CLIENT, 0.4 ms) + - [ops] INSERT (CLIENT, 0.5 ms) + - [ops] create attachments (PRODUCER, 0.8 ms) links=1 messenger.job.attempt=6 + - [ops] INSERT (CLIENT, 0.5 ms) + - [ops] NOTIFY (CLIENT, 0.2 ms) + - [dispatcher] publish attachments (PRODUCER, 70.0 ms) messenger.job.attempt=6 + - [dispatcher] UPDATE (CLIENT, 66.1 ms) + - [worker] process attachments.process.v1 (CONSUMER, 145.2 ms) messenger.job.attempt=6 messenger.job.outcome=ready + - [worker] SELECT (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] UPDATE (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f + - [worker] verify source sha256 (INTERNAL, 0.0 ms) + - [worker] imaging (INTERNAL, 125.7 ms) + - [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/original + - [worker] S3 PutObject (CLIENT, 2.0 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/optimized.webp + - [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/thumbnail.jpg + - [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/crop.jpg + - [worker] commit result (INTERNAL, 4.4 ms) + - [worker] SELECT (CLIENT, 0.5 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] INSERT (CLIENT, 0.2 ms) + - [worker] UPDATE (CLIENT, 0.4 ms) + - [worker] UPDATE (CLIENT, 0.3 ms) + - [worker] INSERT (CLIENT, 0.3 ms) + - [worker] cleanup staging (INTERNAL, 4.1 ms) + - [worker] S3 DeleteObject (CLIENT, 2.1 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f + - [worker] UPDATE (CLIENT, 1.8 ms) +replay trace 9417040b8f913c26894caaaf0996e442: create span links -> ['38319800cdce5da47cb2b821dc03bf6e'] (original upload trace 38319800cdce5da47cb2b821dc03bf6e) +PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it +4 passed, 0 failed diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..4a26755 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,124 @@ +{ + "date": "2026-11-01 20:42", + "api_limit_mib": 512, + "api_limit_cpus": 1.0, + "runs": [ + { + "scenario": "10 x ~9.5 MiB PNG (noise)", + "concurrency": 10, + "file_bytes": [ + 9953047 + ], + "total_mib": 94.9, + "all_202": true, + "wall_seconds": 1.95, + "latency_s": { + "p50": 1.77, + "max": 1.95 + }, + "sampler_rounds": 16, + "replicas": [ + { + "replica": "171a473b0d28", + "uploads_served": 5, + "rss_before_mib": 109.6, + "rss_peak_mib": 124.5, + "rss_growth_mib": 14.9, + "rss_peak_sampled_mib": 124.5, + "cgroup_peak_mib": 162.8, + "cgroup_peak_sampled_mib": 160.2 + }, + { + "replica": "1db2fedff78f", + "uploads_served": 5, + "rss_before_mib": 110.5, + "rss_peak_mib": 125.6, + "rss_growth_mib": 15.1, + "rss_peak_sampled_mib": 125.6, + "cgroup_peak_mib": 163.3, + "cgroup_peak_sampled_mib": 160.8 + } + ] + }, + { + "scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", + "concurrency": 10, + "file_bytes": [ + 116714 + ], + "total_mib": 1.1, + "all_202": true, + "wall_seconds": 1.3, + "latency_s": { + "p50": 0.98, + "max": 1.3 + }, + "sampler_rounds": 9, + "replicas": [ + { + "replica": "171a473b0d28", + "uploads_served": 5, + "rss_before_mib": 107.6, + "rss_peak_mib": 112.4, + "rss_growth_mib": 4.8, + "rss_peak_sampled_mib": 112.4, + "cgroup_peak_mib": 101.5, + "cgroup_peak_sampled_mib": 101.4 + }, + { + "replica": "1db2fedff78f", + "uploads_served": 5, + "rss_before_mib": 107.1, + "rss_peak_mib": 109.8, + "rss_growth_mib": 2.7, + "rss_peak_sampled_mib": 109.8, + "cgroup_peak_mib": 100.0, + "cgroup_peak_sampled_mib": 100.0 + } + ] + }, + { + "scenario": "20 mixed: large + 40 Mpx + JPEG", + "concurrency": 20, + "file_bytes": [ + 1492, + 116714, + 9953047 + ], + "total_mib": 133.2, + "all_202": true, + "wall_seconds": 1.95, + "latency_s": { + "p50": 1.45, + "max": 1.95 + }, + "sampler_rounds": 11, + "replicas": [ + { + "replica": "171a473b0d28", + "uploads_served": 10, + "rss_before_mib": 109.0, + "rss_peak_mib": 136.3, + "rss_growth_mib": 27.3, + "rss_peak_sampled_mib": 136.3, + "cgroup_peak_mib": 191.4, + "cgroup_peak_sampled_mib": 191.4 + }, + { + "replica": "1db2fedff78f", + "uploads_served": 10, + "rss_before_mib": 110.6, + "rss_peak_mib": 135.0, + "rss_growth_mib": 24.3, + "rss_peak_sampled_mib": 135.0, + "cgroup_peak_mib": 172.8, + "cgroup_peak_sampled_mib": 171.4 + } + ] + } + ], + "single_file_extra_rss_mib": { + "naive": 161, + "streaming": 0 + } +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 0000000..c10597c --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,13 @@ +# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas +# date: 2026-11-01 20:42:33, commit: 470d9d6 + uncommitted changes +# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json +# exit code: 0, 53.4 s + +client: Python 3.11.1 on Windows 10 +preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... +large: 9953047 bytes; 40 Mpx: 116714 bytes +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.77, "max": 1.95}, "sampler_rounds": 16, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 109.6, "rss_peak_mib": 124.5, "rss_growth_mib": 14.9, "rss_peak_sampled_mib": 124.5, "cgroup_peak_mib": 162.8, "cgroup_peak_sampled_mib": 160.2}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 110.5, "rss_peak_mib": 125.6, "rss_growth_mib": 15.1, "rss_peak_sampled_mib": 125.6, "cgroup_peak_mib": 163.3, "cgroup_peak_sampled_mib": 160.8}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.3, "latency_s": {"p50": 0.98, "max": 1.3}, "sampler_rounds": 9, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 107.6, "rss_peak_mib": 112.4, "rss_growth_mib": 4.8, "rss_peak_sampled_mib": 112.4, "cgroup_peak_mib": 101.5, "cgroup_peak_sampled_mib": 101.4}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 107.1, "rss_peak_mib": 109.8, "rss_growth_mib": 2.7, "rss_peak_sampled_mib": 109.8, "cgroup_peak_mib": 100.0, "cgroup_peak_sampled_mib": 100.0}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.45, "max": 1.95}, "sampler_rounds": 11, "replicas": [{"replica": "171a473b0d28", "uploads_served": 10, "rss_before_mib": 109.0, "rss_peak_mib": 136.3, "rss_growth_mib": 27.3, "rss_peak_sampled_mib": 136.3, "cgroup_peak_mib": 191.4, "cgroup_peak_sampled_mib": 191.4}, {"replica": "1db2fedff78f", "uploads_served": 10, "rss_before_mib": 110.6, "rss_peak_mib": 135.0, "rss_growth_mib": 24.3, "rss_peak_sampled_mib": 135.0, "cgroup_peak_mib": 172.8, "cgroup_peak_sampled_mib": 171.4}]} +one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB +PASS: all uploads succeeded; largest RSS growth of one replica: 27.3 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/faults.py b/faults.py new file mode 100644 index 0000000..2bb92a7 --- /dev/null +++ b/faults.py @@ -0,0 +1,51 @@ +"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1). + +Тест создаёт в общем volume /faults файлы <точка>., по одному на срабатывание: + + docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1' + +Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса) +и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски. + +Точки: + dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish) + dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана + worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет + worker_crash_after_commit — COMMIT результата был, ack ещё нет + worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease + worker_s3_error — временная ошибка S3 при записи результата + imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения +""" +import logging +import os +import sys +from pathlib import Path + +import settings + +log = logging.getLogger("messenger.faults") + + +def hit(point: str) -> bool: + if not settings.FAULT_INJECTION: + return False + directory = Path(settings.FAULTS_DIR) + try: + candidates = sorted(directory.glob(point + ".*")) + except OSError: + return False + for path in candidates: + try: + path.unlink() + except FileNotFoundError: + continue # этот файл забрал другой процесс + log.warning("FAULT INJECTION: %s", point) + return True + return False + + +def crash(point: str) -> None: + """Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений.""" + if hit(point): + sys.stderr.flush() + os._exit(137) diff --git a/imaging.py b/imaging.py new file mode 100644 index 0000000..a08fbb0 --- /dev/null +++ b/imaging.py @@ -0,0 +1,209 @@ +"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами. + + python imaging.py + +Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя +ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память, +IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker +убивает его по таймауту JOB_TIMEOUT_SECONDS. +Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру. + +Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout. +Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти; +прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной. + +Правила вариантов (contracts/README.md): +- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз + (минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160. +- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются). +- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется. +Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное +отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job +пишет в те же ключи S3 те же байты. +""" +import json +import os +import resource +import sys +import time + + +def apply_limits() -> None: + """Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов.""" + memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024 + cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60")) + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024)) + + +if __name__ == "__main__": + apply_limits() + +from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом + +PIPELINE_VERSION = 1 +THUMB_BOX = 256 +CROP_SIZE = 128 +OPTIMIZED_BOX = 1280 +THUMB_QUALITY = 85 +CROP_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +EXIT_INVALID = 3 +EXIT_TOO_LARGE = 4 +EXIT_MEMORY = 5 + +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose) +ORIENTATION = { + 2: [Image.Transpose.FLIP_LEFT_RIGHT], + 3: [Image.Transpose.ROTATE_180], + 4: [Image.Transpose.FLIP_TOP_BOTTOM], + 5: [Image.Transpose.TRANSPOSE], + 6: [Image.Transpose.ROTATE_270], + 7: [Image.Transpose.TRANSVERSE], + 8: [Image.Transpose.ROTATE_90], +} + + +class TooLarge(Exception): + pass + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + """Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def orientation_of(image: Image.Image) -> int: + try: + value = image.getexif().get(0x0112, 1) + except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем + return 1 + return value if value in ORIENTATION else 1 + + +def normalize(image: Image.Image) -> Image.Image: + """Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность).""" + mode = image.mode + transparent = "transparency" in image.info + if mode in ("I;16", "I;16B", "I;16L", "I"): + image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L") + mode = "L" + if mode == "1": + return image.convert("L") + if mode in ("P", "PA"): + return image.convert("RGBA" if transparent or mode == "PA" else "RGB") + if mode in ("LA", "La", "RGBa"): + return image.convert("RGBA") + if mode in ("L", "RGB") and transparent: + return image.convert("RGBA") + if mode in ("L", "RGB", "RGBA"): + return image + return image.convert("RGB") # CMYK, YCbCr, LAB, ... + + +def on_white(image: Image.Image) -> Image.Image: + """Прозрачность -> белый фон (для JPEG).""" + if image.mode != "RGBA": + return image + background = Image.new("RGB", image.size, (255, 255, 255)) + background.paste(image, mask=image.getchannel("A")) + return background + + +def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict: + Image.MAX_IMAGE_PIXELS = max_pixels + with Image.open(source) as image: + if image.format != FORMATS.get(content_type): + raise ValueError("format does not match stored content type") + width, height = image.size + if width * height > max_pixels: + raise TooLarge() + orientation = orientation_of(image) + icc = image.info.get("icc_profile") + swap = orientation in (5, 6, 7, 8) + oriented = (height, width) if swap else (width, height) + + # Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128 + # не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной. + target = fit(*oriented, OPTIMIZED_BOX) + stored_target = (target[1], target[0]) if swap else target + if image.format == "JPEG": + image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает + image.load() + work = normalize(image) + if work.size != stored_target: + work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0) + elif work is image: + work = work.copy() + for step in ORIENTATION.get(orientation, []): + work = work.transpose(step) + + extra = {"icc_profile": icc} if icc else {} + results = [] + + optimized = work if work.mode != "L" else work.convert("RGB") + path = os.path.join(out_dir, "optimized.webp") + optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra) + results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width, + "height": optimized.height, "file": "optimized.webp"}) + + thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии + thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS) + thumb = on_white(thumb) + path = os.path.join(out_dir, "thumbnail.jpg") + thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra) + results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width, + "height": thumb.height, "file": "thumbnail.jpg"}) + + side = min(work.size) + left, top = (work.width - side) // 2, (work.height - side) // 2 + crop = work.crop((left, top, left + side, top + side)) + crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)) + path = os.path.join(out_dir, "crop.jpg") + crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra) + results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE, + "file": "crop.jpg"}) + + return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height, + "orientation": orientation}, "variants": results} + + +def injected_fault() -> None: + """Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1).""" + fault = os.environ.get("FAULT_IMAGING", "") + if fault == "hang": + time.sleep(3600) + elif fault == "invalid": + raise ValueError("injected invalid image") + elif fault == "oom": + blocks = [] + while True: + blocks.append(bytearray(64 * 1024 * 1024)) + + +def main() -> int: + source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) + try: + injected_fault() + result = process(source, out_dir, content_type, max_pixels) + except (TooLarge, Image.DecompressionBombError): + return EXIT_TOO_LARGE + except MemoryError: + return EXIT_MEMORY + except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError): + # Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу. + return EXIT_INVALID + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..b350588 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,2288 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — lab6 observability", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Обзор и SLO (окно SLO — 1 час)", + "id": 2, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "id": 3, + "title": "RPS API", + "description": "Бизнес-запросы API в секунду (без /health), все реплики.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_requests:rate1m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 4, + "title": "Доля 5xx (1m)", + "description": "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_error_ratio:rate1m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 5, + "title": "p95 API (5m)", + "description": "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_request_duration_seconds:p95_5m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 6, + "title": "Старейшее задание", + "description": "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process.v1\"})", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 7, + "title": "SLO-1 доступность", + "description": "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:ratio_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 8, + "title": "Бюджет ошибок SLO-1", + "description": "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:error_budget_remaining_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0 + }, + { + "color": "green", + "value": 0.5 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 9, + "title": "SLO-2 POST < 250 мс", + "description": "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_latency_messages:ratio_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 10, + "title": "SLO-3 ready ≤ 30 с", + "description": "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_time_to_ready:ratio_1h", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + } + }, + { + "type": "table", + "id": 11, + "title": "Алерты Prometheus (pending/firing)", + "description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "ALERTS{alertstate=~\"pending|firing\"}", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true + } + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + } + }, + { + "type": "row", + "title": "API: запросы, ошибки, задержка (RED)", + "id": 12, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 13, + "title": "RPS по маршрутам", + "description": "Шаблон маршрута, а не URL: метка ограниченной кардинальности.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "{{http_request_method}} {{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 14, + "title": "RPS по классу статуса", + "description": "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "{{http_response_status_class}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 15, + "title": "Доля 5xx", + "description": "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_response_status_class=\"5xx\"}[$__rate_interval])) or on() vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))", + "legendFormat": "5xx ratio", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 16, + "title": "Задержка API p50 / p95 / p99", + "description": "histogram_quantile по сумме buckets всех реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 17, + "title": "p95 по маршрутам", + "description": "Где именно медленно.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 18, + "title": "Запросы в обработке (по репликам)", + "description": "http.server.active_requests: насыщение реплик API.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (http_server_active_requests{job=\"messenger-api\"})", + "legendFormat": "{{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Фоновая обработка: очередь, worker, время до ready", + "id": 19, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 20, + "title": "Попытки заданий в секунду по исходу", + "description": "ready / retry / failed / stale / lease_lost (messenger.job.duration count).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 21, + "title": "Длительность попытки p50 / p95 / p99 (ready)", + "description": "От claim до COMMIT ready в worker.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"ready\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"ready\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"ready\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 22, + "title": "Время до ready p50 / p95 / p99", + "description": "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{job_type=\"attachment.process.v1\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{job_type=\"attachment.process.v1\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{job_type=\"attachment.process.v1\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 23, + "title": "Глубина очереди и backlog", + "description": "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (messaging_destination_name) (messenger_queue_depth)", + "legendFormat": "broker {{messaging_destination_name}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (status) (messenger_jobs_backlog)", + "legendFormat": "jobs {{status}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 24, + "title": "Возраст старейшего задания и outbox", + "description": "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process.v1\"})", + "legendFormat": "oldest queued job", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_outbox_oldest_age_seconds)", + "legendFormat": "oldest unpublished outbox row", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 25, + "title": "Доставки и публикации", + "description": "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", + "legendFormat": "consumed {{outcome}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", + "legendFormat": "published {{outcome}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Насыщение: пул БД, worker, CPU/RAM", + "id": 26, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 27, + "title": "Пул PostgreSQL API: занято / максимум / ждут", + "description": "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (db_client_connection_count{db_client_connection_state=\"used\"})", + "legendFormat": "used {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(db_client_connection_max)", + "legendFormat": "max per replica", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (db_client_connection_pending_requests)", + "legendFormat": "waiting {{instance}}", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 28, + "title": "Ожидание соединения из пула p95", + "description": "Время получения соединения из пула API (не время запроса).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95 {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 29, + "title": "Worker: занятость (busy / capacity)", + "description": "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)", + "legendFormat": "saturation", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 30, + "title": "CPU процессов (ядра)", + "description": "rate(process.cpu.time): ядер на процесс.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 53 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 31, + "title": "CPU контейнеров / лимит", + "description": "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 53 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 32, + "title": "Память: RSS процесса и контейнер", + "description": "process.memory.usage (RSS) и container.memory.usage (cgroup).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 53 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (job, instance) (process_memory_usage_bytes)", + "legendFormat": "rss {{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (job, instance) (container_memory_usage_bytes)", + "legendFormat": "cgroup {{job}} {{instance}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Конвейер телеметрии: Collector и SDK", + "id": 33, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 34, + "title": "Collector: принято и отправлено", + "description": "Спаны/записи логов/точки метрик в секунду.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 62 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", + "legendFormat": "accepted spans", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", + "legendFormat": "accepted logs", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", + "legendFormat": "accepted metric points", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", + "legendFormat": "sent spans {{exporter}}", + "refId": "D", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", + "legendFormat": "sent logs {{exporter}}", + "refId": "E", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 35, + "title": "Collector: потери и отказы", + "description": "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 62 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", + "legendFormat": "send failed spans {{exporter}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", + "legendFormat": "send failed logs {{exporter}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", + "legendFormat": "queue full spans {{exporter}}", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", + "legendFormat": "refused spans (memory_limiter)", + "refId": "D", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 36, + "title": "Очереди exporter и RSS Collector", + "description": "Заполнение sending_queue (доля ёмкости) и память процесса Collector.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 62 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", + "legendFormat": "queue {{exporter}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 37, + "title": "Collector RSS", + "description": "Ограничено memory_limiter (200 MiB) при недоступном хранилище.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(otelcol_process_memory_rss)", + "legendFormat": "collector rss", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 38, + "title": "SDK приложений: экспорт и потери", + "description": "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=\"\"}[$__rate_interval]))", + "legendFormat": "exported spans {{job}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "export failed spans {{job}} {{error_type}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "export failed logs {{job}} {{error_type}}", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "queue full spans {{job}}", + "refId": "D", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Логи и трассы", + "id": 39, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "panels": [] + }, + { + "type": "logs", + "id": 40, + "title": "Предупреждения и ошибки (Loki)", + "description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 79 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "queryType": "range", + "expr": "{service_name=~\"messenger-.*\"} | detected_level=~\"warn|error\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + } + }, + { + "type": "table", + "id": 41, + "title": "Трассы с ошибками (Tempo, TraceQL)", + "description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 79 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "query": "{ status = error }", + "limit": 20, + "tableType": "traces" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + } + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..ac76d2b --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +# Grafana provisioning: dashboard из файлов (infra/grafana/dashboards/*.json монтируется только для чтения). +apiVersion: 1 + +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..9ccd6d5 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,67 @@ +# Grafana provisioning: источники данных (импортируются при старте, ручной настройки нет). +# Grafana сама ничего не хранит из трёх сигналов — только интерфейс и запросы к хранилищам. +# "$$" — буквальный "$" (иначе Grafana подставит переменную окружения). +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + prometheusType: Prometheus + prometheusVersion: 3.15.1 + timeInterval: 15s + manageAlerts: false # правила алертов в Git (infra/prometheus/rules), Grafana их только показывает + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + # trace_id — structured metadata записи (OTLP): ссылка «лог -> трасса» в Tempo. + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: Открыть трассу + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + # «спан -> логи»: те же trace_id в Loki, окно ±5 минут вокруг спана. + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: false + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__trace.traceId}"' + # «спан -> метрики»: задержка и ошибки сервиса за время спана. + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + queries: + - name: API p95 + query: 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))' + nodeGraph: + enabled: true + search: + hide: false + lokiSearch: + datasourceUid: loki diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..1605eed --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,77 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + # Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка + # при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт, + # сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT). + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..9845653 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,52 @@ +# Loki 3.7.9 — хранилище ЛОГОВ (single binary, файловая система в volume lokidata). +# Логи приходят от Collector по OTLP (/otlp/v1/logs). Ресурсные атрибуты service.name, +# service.instance.id, deployment.environment.name становятся индексными метками (service_name, ...); +# атрибуты записи (event, request_id, trace_id, span_id, severity) — structured metadata, по ним +# фильтруют без индекса: {service_name="messenger-api"} | trace_id="". +# Проверка: loki -config.file=loki.yaml -verify-config +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true # нужно для OTLP: trace_id/span_id и атрибуты записи + retention_period: 168h # 7 дней учебной истории + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_query_lookback: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +pattern_ingester: + enabled: false + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..40541b2 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,129 @@ +# OpenTelemetry Collector (otel/opentelemetry-collector-contrib:0.164.0) — единственный приёмник +# телеметрии приложения. Не хранит данные: принимает OTLP, ограничивает память, пакетирует и +# отправляет в три хранилища. Проверка конфигурации: otelcol-contrib validate --config=collector.yaml +# +# api/worker/dispatcher/janitor --OTLP/HTTP :4318--> receiver otlp +# traces -> memory_limiter -> attributes/scrub -> batch -> otlp_grpc/tempo (Tempo :4317) +# logs -> memory_limiter -> attributes/scrub -> batch -> otlp_http/loki (Loki /otlp) +# metrics -> memory_limiter -> batch -> otlp_http/prometheus (Prometheus /api/v1/otlp) +# +# Порты 4317/4318/8888/13133 слушаются только во внутренней сети telemetry (в compose не публикуются). +# +# Поведение при недоступном хранилище (ограниченная буферизация, RAM не растёт бесконечно): +# - у каждого exporter своя очередь в памяти (sending_queue) фиксированного размера: заполнилась -> +# новые пакеты отбрасываются и считаются (otelcol_exporter_enqueue_failed_*); +# - повтор с экспоненциальной задержкой не дольше max_elapsed_time, потом пакет отбрасывается +# (otelcol_exporter_send_failed_*); +# - memory_limiter: при приближении к лимиту receiver отказывает клиентам (SDK получает ошибку и +# сам отбрасывает данные в своей ограниченной очереди), процесс не уходит в OOM; +# - остальные pipeline работают независимо: упал Tempo — логи и метрики продолжают доходить. +# Очередь только в памяти: при рестарте Collector неотправленное теряется (осознанный компромисс, +# file_storage не нужен для учебного стенда). + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + # Первым в каждом pipeline: проверка heap Go раз в секунду. Мягкий предел = limit - spike = 160 MiB + # (receiver начинает отказывать), жёсткий 200 MiB (принудительный GC). Лимит контейнера 384 MiB: + # RSS включает ещё runtime Go и отображённый в память бинарник contrib (~150 MiB, вытесняемые страницы). + memory_limiter: + check_interval: 1s + limit_mib: 200 + spike_limit_mib: 40 + + # Защита в глубину: приложение не пишет секреты в атрибуты, но если бы инструментирование ошиблось, + # такие ключи не дойдут до хранилищ. Удаляются по шаблону имени ключа (спаны и логи). + attributes/scrub: + actions: + - pattern: (?i).*(password|passwd|authorization|cookie|secret|token|api[_.-]?key|signature).* + action: delete + - key: db.statement.parameters + action: delete + - key: db.query.parameter + action: delete + + batch: + timeout: 2s + send_batch_size: 1024 + send_batch_max_size: 2048 + +exporters: + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true # lab6: внутренняя сеть; TLS/mTLS по trust matrix — в lab7 + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 # пакетов (до 2048 спанов каждый) + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 120s + + otlp_http/loki: + endpoint: http://loki:3100/otlp # -> /otlp/v1/logs + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 120s + + otlp_http/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp # -> /api/v1/otlp/v1/metrics + timeout: 5s + sending_queue: + enabled: true + num_consumers: 1 # один поток: точки одной серии приходят по порядку + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 120s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + # Собственные метрики Collector (принято/отправлено/отброшено, размер очередей, RSS) — + # Prometheus забирает их pull-ом с otel-collector:8888. Так потери телеметрии видны и считаются. + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_grpc/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_http/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/prometheus] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..f0321b7 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +# Prometheus 3.15.1 — хранилище МЕТРИК. Принимает метрики приложения push-ом по OTLP от Collector +# (флаг --web.enable-otlp-receiver, путь /api/v1/otlp/v1/metrics) и сам забирает (pull) служебные +# метрики Collector, Loki и Tempo. Вычисляет recording rules (SLI) и alert rules; состояние алертов +# (pending/firing/resolved) видно на /alerts и через /api/v1/alerts — внешняя доставка не нужна. +# Проверка: promtool check config prometheus.yml +global: + scrape_interval: 15s + scrape_timeout: 5s + evaluation_interval: 15s + +# Как имена OTLP переводятся в имена Prometheus (фиксируем явно, чтобы dashboard не сломался при +# смене значения по умолчанию): точки -> подчёркивания, суффиксы единиц и _total. +# http.server.request.duration (s, histogram) -> http_server_request_duration_seconds_{bucket,sum,count} +# атрибут http.route -> метка http_route; service.name -> job; service.instance.id -> instance. +otlp: + translation_strategy: UnderscoreEscapingWithSuffixes + +storage: + tsdb: + # Collector после недоступности Prometheus дошлёт пакеты из очереди с прошлыми временными метками. + out_of_order_time_window: 10m + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + # Собственные метрики Collector: принято/отправлено/отброшено по сигналам, очереди, RSS. + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] diff --git a/infra/prometheus/rules/alerts.yml b/infra/prometheus/rules/alerts.yml new file mode 100644 index 0000000..1606ac3 --- /dev/null +++ b/infra/prometheus/rules/alerts.yml @@ -0,0 +1,113 @@ +# Alert rules (Prometheus). Внешней доставки нет: состояние inactive/pending/firing видно на +# http://127.0.0.1:9090/alerts, через GET /api/v1/alerts и в Grafana (Alerting -> data source rules); +# метрика ALERTS{alertstate="firing"} хранится в TSDB, поэтому видно и когда алерт стал resolved. +# Каждое правило: условие, окно rate, порог, for, описание и ссылка на локальный runbook. +# Проверка: promtool check rules alerts.yml; promtool test rules ../tests/alerts_test.yml +groups: + - name: messenger-alerts + interval: 15s + rules: + # (1) Высокая доля ошибок API. Окно 1m, порог 5% (в 5 раз выше бюджета SLO-1 = 1%), for 1m, + # и не меньше 0,2 запроса/с, чтобы единичный 503 при пустом трафике не будил дежурного. + - alert: MessengerHighErrorRate + expr: | + ( + (sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) or on() vector(0)) + / sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) + ) > 0.05 + and on() + sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) > 0.2 + for: 1m + labels: + severity: page + service: messenger-api + slo: availability + annotations: + summary: "API: {{ $value | humanizePercentage }} бизнес-запросов завершаются 5xx (порог 5%)" + description: >- + Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше + минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка + кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу. + runbook_url: docs/runbooks.md#messengerhigherrorrate + + # (2) Растущий lag очереди: возраст старейшего ГОТОВОГО к обработке задания (available_at прошёл, + # статус queued) больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно, + # так что правило ловит именно рост, а не единичный всплеск глубины очереди. + - alert: MessengerQueueLagGrowing + expr: max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"}) > 30 + for: 1m + labels: + severity: page + service: messenger-worker + slo: time_to_ready + annotations: + summary: "Очередь attachments.process.v1: старейшее задание ждёт {{ $value | humanizeDuration }}" + description: >- + Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с + дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди, + занятость worker (busy/capacity) и ошибки worker в логах. + runbook_url: docs/runbooks.md#messengerqueuelaggrowing + + # (3) Ни одного consumer у очереди: worker остановлен/упал. Дополняет (2): срабатывает даже без нагрузки. + - alert: MessengerWorkerNoConsumers + expr: max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}) == 0 + for: 1m + labels: + severity: warning + service: messenger-worker + annotations: + summary: "У очереди attachments.process.v1 нет consumers" + description: "RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру." + runbook_url: docs/runbooks.md#messengerworkernoconsumers + + # (4) Насыщение пула соединений API: запросы ждут соединение дольше минуты. + - alert: MessengerDbPoolSaturated + expr: max by (instance) (db_client_connection_pending_requests{job="messenger-api"}) > 0 + for: 1m + labels: + severity: warning + service: messenger-api + annotations: + summary: "Пул PostgreSQL реплики {{ $labels.instance }}: {{ $value }} запросов ждут соединение" + description: "Все DB_POOL_MAX соединений заняты. Причина обычно — медленные запросы или блокировки в PostgreSQL, а не нехватка соединений." + runbook_url: docs/runbooks.md#messengerdbpoolsaturated + + # (5) Collector не может доставить телеметрию в Tempo/Loki/Prometheus: либо пакеты уже теряются + # (send_failed — истёк повтор 120 с у пакета, enqueue_failed — очередь полна), либо очередь exporter + # дольше минуты заполнена больше чем на 5% (хранилище не принимает, пакеты копятся). Во время пакет + # ждёт в очереди его окно повторов не идёт, поэтому без второго условия алерт сработал бы только + # после первых потерь, то есть через 2+ минуты недоступности. + - alert: TelemetryExportFailing + expr: | + sum by (exporter) ( + rate(otelcol_exporter_send_failed_spans[2m]) + or rate(otelcol_exporter_send_failed_log_records[2m]) + or rate(otelcol_exporter_send_failed_metric_points[2m]) + or rate(otelcol_exporter_enqueue_failed_spans[2m]) + or rate(otelcol_exporter_enqueue_failed_log_records[2m]) + or rate(otelcol_exporter_enqueue_failed_metric_points[2m]) + ) > 0 + or + max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.05 + for: 1m + labels: + severity: warning + service: otel-collector + annotations: + summary: "Collector не доставляет телеметрию: exporter {{ $labels.exporter }}" + description: "Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*." + runbook_url: docs/runbooks.md#telemetryexportfailing + + # (6) Collector недоступен: Prometheus не может забрать его служебные метрики. Метрики приложения + # в этот период не доходят (их везёт Collector), поэтому алерт смотрит на up самого Collector. + - alert: TelemetryCollectorDown + expr: up{job="otel-collector"} == 0 + for: 1m + labels: + severity: warning + service: otel-collector + annotations: + summary: "OTel Collector недоступен" + description: "Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском." + runbook_url: docs/runbooks.md#telemetrycollectordown + diff --git a/infra/prometheus/rules/recording.yml b/infra/prometheus/rules/recording.yml new file mode 100644 index 0000000..46ad07b --- /dev/null +++ b/infra/prometheus/rules/recording.yml @@ -0,0 +1,50 @@ +# Recording rules: SLI и агрегаты для dashboard. Проверка: promtool check rules recording.yml +# +# Числитель/знаменатель ошибок API (contracts/observability.md): +# знаменатель — все бизнес-запросы API (job="messenger-api", маршрут НЕ /health/*), любой код; +# числитель — из них ответы 5xx (отказ сервера: БД/Redis/S3 недоступны, внутренняя ошибка). +# 4xx (неверный пароль, 404 чужого чата, 409) — поведение клиента, в ошибки SLO не входят. +# Percentile — только из buckets гистограммы, просуммированных по всем репликам ДО histogram_quantile +# (sum by (le)); p95 отдельных реплик не усредняются. +groups: + - name: messenger-api-sli + interval: 15s + rules: + - record: messenger:http_requests:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) + - record: messenger:http_errors_5xx:rate1m + expr: | + sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) + or on() vector(0) + - record: messenger:http_error_ratio:rate1m + expr: messenger:http_errors_5xx:rate1m / messenger:http_requests:rate1m + - record: messenger:http_request_duration_seconds:p50_5m + expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:http_request_duration_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + - record: messenger:http_request_duration_seconds:p99_5m + expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m]))) + + # SLO (учебное скользящее окно 1 ч; в эксплуатации то же выражение с [28d]): + # SLO-1 доступность: >= 99% бизнес-запросов без 5xx; + # SLO-2 задержка: >= 95% POST /api/v1/chats/{chat_id}/messages быстрее 250 мс; + # SLO-3 фоновая обработка: >= 95% вложений ready не позже 30 с после приёма (202). + - name: messenger-slo + interval: 30s + rules: + - record: messenger:slo_availability:ratio_1h + expr: | + 1 - ( + (sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1h])) or on() vector(0)) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1h])) + ) + - record: messenger:slo_availability:error_budget_remaining_1h + expr: 1 - (1 - messenger:slo_availability:ratio_1h) / (1 - 0.99) + - record: messenger:slo_latency_messages:ratio_1h + expr: | + sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx", le="0.25"}[1h])) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}[1h])) + - record: messenger:slo_time_to_ready:ratio_1h + expr: | + sum(increase(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) + / sum(increase(messenger_attachment_time_to_ready_seconds_count[1h])) diff --git a/infra/prometheus/tests/alerts_test.yml b/infra/prometheus/tests/alerts_test.yml new file mode 100644 index 0000000..dc660cf --- /dev/null +++ b/infra/prometheus/tests/alerts_test.yml @@ -0,0 +1,121 @@ +# Unit-тесты alert rules на синтетических рядах (без стенда): +# promtool test rules infra/prometheus/tests/alerts_test.yml +# docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml +# Проверяют порог, окно и for: правило не срабатывает раньше времени и при пустом трафике. +rule_files: + - ../rules/alerts.yml +evaluation_interval: 15s + +tests: + # MessengerHighErrorRate: 2 rps успешных, с 3-й минуты +1 rps 5xx (доля 33%). + - interval: 15s + input_series: + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}' + values: '0+30x60' + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="5xx"}' + values: '0x12 0+15x48' + # health-проверки с ошибками не считаются (знаменатель и числитель без /health) + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/health/ready", http_response_status_class="5xx"}' + values: '0+100x60' + alert_rule_test: + - eval_time: 3m + alertname: MessengerHighErrorRate + exp_alerts: [] + - eval_time: 3m45s # условие выполняется, но for: 1m ещё не прошёл + alertname: MessengerHighErrorRate + exp_alerts: [] + - eval_time: 5m + alertname: MessengerHighErrorRate + exp_alerts: + - exp_labels: {severity: page, service: messenger-api, slo: availability} + exp_annotations: + summary: 'API: 33.33% бизнес-запросов завершаются 5xx (порог 5%)' + description: Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу. + runbook_url: docs/runbooks.md#messengerhigherrorrate + + # Пустой трафик: единичные 5xx при 0,05 rps не будят дежурного (порог 0,2 запроса/с). + - interval: 15s + input_series: + - series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/users/me", http_response_status_class="5xx"}' + values: '0+1x40' + alert_rule_test: + - eval_time: 9m + alertname: MessengerHighErrorRate + exp_alerts: [] + + # MessengerQueueLagGrowing: возраст старейшего задания растёт со 2-й минуты, затем worker вернулся. + - interval: 15s + input_series: + - series: 'messenger_queue_oldest_age_seconds{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}' + values: '0x8 0+15x20 0x20' + - series: 'messenger_queue_consumers{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}' + values: '2x8 0x20 2x20' + alert_rule_test: + - eval_time: 2m30s # возраст 30 с — ещё не больше порога + alertname: MessengerQueueLagGrowing + exp_alerts: [] + - eval_time: 4m + alertname: MessengerQueueLagGrowing + exp_alerts: + - exp_labels: {severity: page, service: messenger-worker, slo: time_to_ready} + exp_annotations: + summary: 'Очередь attachments.process.v1: старейшее задание ждёт 1m 45s' + description: 'Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди, занятость worker (busy/capacity) и ошибки worker в логах.' + runbook_url: docs/runbooks.md#messengerqueuelaggrowing + - eval_time: 4m + alertname: MessengerWorkerNoConsumers + exp_alerts: + - exp_labels: {severity: warning, service: messenger-worker} + exp_annotations: + summary: У очереди attachments.process.v1 нет consumers + description: 'RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру.' + runbook_url: docs/runbooks.md#messengerworkernoconsumers + - eval_time: 9m # worker вернулся, очередь разобрана -> resolved + alertname: MessengerQueueLagGrowing + exp_alerts: [] + + # TelemetryExportFailing и TelemetryCollectorDown + - interval: 15s + input_series: + - series: 'otelcol_exporter_send_failed_spans{exporter="otlp_grpc/tempo"}' + values: '0x8 0+50x20' + - series: 'up{job="otel-collector"}' + values: '1x8 0x20' + alert_rule_test: + - eval_time: 5m + alertname: TelemetryExportFailing + exp_alerts: + - exp_labels: {severity: warning, service: otel-collector, exporter: otlp_grpc/tempo} + exp_annotations: + summary: 'Collector не доставляет телеметрию: exporter otlp_grpc/tempo' + description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.' + runbook_url: docs/runbooks.md#telemetryexportfailing + - eval_time: 5m + alertname: TelemetryCollectorDown + exp_alerts: + - exp_labels: {severity: warning, service: otel-collector, job: otel-collector} + exp_annotations: + summary: OTel Collector недоступен + description: Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском. + runbook_url: docs/runbooks.md#telemetrycollectordown + + # TelemetryExportFailing по заполнению очереди: Loki недоступен, потерь ещё нет, но очередь растёт. + - interval: 15s + input_series: + - series: 'otelcol_exporter_queue_size{exporter="otlp_http/loki"}' + values: '0x8 0+2x30' + - series: 'otelcol_exporter_queue_capacity{exporter="otlp_http/loki"}' + values: '200x38' + alert_rule_test: + - eval_time: 3m # 6 пакетов из 200 (3%) — ниже порога 5% + alertname: TelemetryExportFailing + exp_alerts: [] + - eval_time: 5m + alertname: TelemetryExportFailing + exp_alerts: + - exp_labels: {severity: warning, service: otel-collector, exporter: otlp_http/loki} + exp_annotations: + summary: 'Collector не доставляет телеметрию: exporter otlp_http/loki' + description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.' + runbook_url: docs/runbooks.md#telemetryexportfailing + diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 0000000..f4f41e0 --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,22 @@ +# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json +# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет: +# при импорте определений на старте пользователь по умолчанию не создаётся. +definitions.import_backend = local_filesystem +definitions.local.path = /run/secrets/rabbitmq_definitions +definitions.skip_if_unchanged = true + +listeners.tcp.default = 5672 +loopback_users = none + +# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет), +# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста). +vm_memory_high_watermark.absolute = 320MB +disk_free_limit.absolute = 256MB + +# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение +# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3. +consumer_timeout = 300000 + +log.console = true +log.console.level = info +log.file = false diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 0000000..33c8c08 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..aeebf40 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,39 @@ +# Tempo 3.1.1 — хранилище ТРАСС (single binary: distributor -> live-store -> блоки в volume tempodata, +# без Kafka). Трассы приходят от Collector по OTLP gRPC :4317 (только внутренняя сеть telemetry), +# запросы Grafana и тестов — HTTP :3200 (/api/traces/, /api/search, TraceQL). +# Проверка: tempo -config.file=tempo.yaml -config.verify=true +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + grpc_listen_port: 9095 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +# Хранение 7 дней (как у Loki); удаление старых блоков делают backend-scheduler/worker. +backend_scheduler: + local_work_path: /var/tempo/scheduler + provider: + compaction: + compaction: + block_retention: 168h +backend_worker: + compaction: + block_retention: 168h + +usage_report: + reporting_enabled: false diff --git a/janitor.py b/janitor.py new file mode 100644 index 0000000..904a2fb --- /dev/null +++ b/janitor.py @@ -0,0 +1,322 @@ +"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor). + + python janitor.py --once [--dry-run] [--grace-seconds N] + +Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock): + +1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен + в очередь. Удалить объект staging (идемпотентно), затем строку. +2. Staging после безопасной фиксации: + - ready: исходник уже опубликован в attachments//v/original -> удалить staging-объект + (обычно это делает worker сразу после ack; здесь — если он не успел); + - failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется. + queued/processing не трогаются никогда: это текущая работа. +3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются. +4. Обход bucket по всем prefix (объекты старше grace): + - staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key; + - attachments//...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен: + вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention); + оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing + и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их + теми же ключами, и удаление устаревшим проходом сломало бы новый результат. + Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения: + replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним. + Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных. + +Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому +janitor не удаляет то, что ещё пишется. +""" +import argparse +import json +import logging +import re +import signal +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-janitor") + +import pipeline # noqa: E402 +import storage # noqa: E402 + +log = logging.getLogger("messenger.janitor") + +JANITOR_LOCK = 20260404 +BATCH = 100 +STOP = threading.Event() +ACTOR = "janitor" +FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/") + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def idle(seconds: float) -> bool: + """Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки.""" + deadline = time.monotonic() + seconds + while (left := deadline - time.monotonic()) > 0: + touch_alive() + if STOP.wait(min(left, 10)): + return True + return False + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: + """Шаг 1. Возвращает (найдено, удалено).""" + found = deleted = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, source_key FROM attachments + WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + FOR UPDATE SKIP LOCKED""", + (grace, BATCH), + ).fetchall() + found += len(rows) + if dry_run or not rows: + return found, deleted + for row in rows: + storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) + deleted += 1 + log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) + if len(rows) < BATCH: + return found, deleted + return found, deleted + + +def cleanup_staging(conn, dry_run: bool) -> dict: + """Шаг 2. Исходники ready (сразу) и failed (после retention).""" + report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0} + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a + LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.source_key IS NOT NULL + AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds') + OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s))) + ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""", + (settings.STAGING_RETENTION_SECONDS, BATCH), + ).fetchall() + if dry_run or not rows: + if dry_run: + report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows) + report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows) + break + for row in rows: + storage.delete(row["source_key"]) + conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],)) + if row["state"] == "ready": + report["staging_ready_deleted"] += 1 + else: + report["staging_failed_expired"] += 1 + if row["job_id"] is not None: + pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR, + "replay no longer possible") + if len(rows) < BATCH: + break + report["staging_kept_for_replay"] = conn.execute( + "SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"] + return report + + +def cleanup_outbox(conn, dry_run: bool) -> int: + if dry_run: + return conn.execute( + """SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"] + return conn.execute( + """DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).rowcount + + +def classify_final(conn, keys: list[str]) -> set[str]: + """Какие ключи attachments//... не принадлежат ни одному живому вложению (кандидаты в сироты).""" + ids = {} + for key in keys: + match = FINAL_KEY_RE.match(key) + if match: + try: + ids[key] = UUID(match.group(1)) + except ValueError: + pass + unique = list(set(ids.values())) + rows = {r["id"]: r for r in conn.execute( + """SELECT a.id, a.state, a.object_key, a.source_key, + ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()} + orphans = set() + for key in keys: + attachment_id = ids.get(key) + row = rows.get(attachment_id) if attachment_id else None + if row is None: + orphans.add(key) + elif row["state"] == "failed" and row["source_key"] is None: + # failed без исходника — конечное состояние: replay отказывает («source file is no longer in + # staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи. + orphans.add(key) + elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]: + orphans.add(key) + return orphans + + +def sweep_objects(conn, grace: int, dry_run: bool) -> dict: + """Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" + started = datetime.now(timezone.utc) + cutoff = started - timedelta(seconds=grace) + scanned = orphans = orphans_deleted = 0 + listed = set() + + def orphan_keys(prefix, items): + keys = [key for key, _ in items] + if prefix == storage.FINAL_PREFIX: + return classify_final(conn, keys) + known = {r["source_key"] for r in conn.execute( + "SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()} + return {key for key in keys if key not in known} + + def delete_if_orphan(prefix, key) -> bool: + """Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker.""" + if prefix != storage.FINAL_PREFIX: + # staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется, + # поэтому «нет строки сейчас» означает «не будет никогда». + if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None: + return False + storage.delete(key) + return True + match = FINAL_KEY_RE.match(key) + try: + attachment_id = UUID(match.group(1)) if match else None + except ValueError: + attachment_id = None + with conn.transaction(): + if attachment_id is not None: + # FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но + # конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT. + conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,)) + if key not in classify_final(conn, [key]): + return False + storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе + return True + + def check(prefix, items): + nonlocal orphans, orphans_deleted + candidates = orphan_keys(prefix, items) + for key, modified in items: + if key not in candidates or modified >= cutoff: + continue + orphans += 1 + if not dry_run and delete_if_orphan(prefix, key): + orphans_deleted += 1 + log.info("deleted orphan object %s", key) + + for prefix in storage.ALL_PREFIXES: + page = [] + for key, modified, _size in storage.list_objects(prefix): + scanned += 1 + listed.add(key) + page.append((key, modified)) + if len(page) >= 1000: + check(prefix, page) + page = [] + if page: + check(prefix, page) + + missing = [] + for r in conn.execute( + """SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall(): + if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]): + missing.append(str(r["id"])) + if missing: + log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s", + ", ".join(missing[:20])) + return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, + "missing_objects": len(missing)} + + +def run_once(grace: int, dry_run: bool) -> dict | None: + # Один проход = одна трасса (раз в JANITOR_INTERVAL_SECONDS): запросы к БД и S3 — её дети. + with telemetry.internal_span("janitor pass", {"messenger.janitor.dry_run": dry_run}) as span: + report = _run_once(grace, dry_run) + if report: + span.set_attributes({f"messenger.janitor.{k}": v for k, v in report.items() + if isinstance(v, (int, float, bool))}) + return report + + +def _run_once(grace: int, dry_run: bool) -> dict | None: + started = time.monotonic() + with connect() as conn: + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: + log.info("another janitor is running, skip") + return None + try: + found, deleted = reclaim_stale_intents(conn, grace, dry_run) + report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, + "stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run), + "outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)} + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) + report["seconds"] = round(time.monotonic() - started, 3) + return report + + +def main(): + parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects") + parser.add_argument("--once", action="store_true", help="one pass and exit") + parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") + parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) + args = parser.parse_args() + if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: + parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " + "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") + + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + while True: + touch_alive() + try: + report = run_once(args.grace_seconds, args.dry_run) + if report is not None: + print(json.dumps(report), flush=True) + log.info("janitor pass done", extra={"event": "janitor.pass", **{ + k: v for k, v in report.items() if isinstance(v, (int, float, bool))}}) + failed = False + except (psycopg.Error, storage.StorageUnavailable) as exc: + log.warning("pass failed, will retry: %s", type(exc).__name__, + extra={"event": "janitor.pass_failed", "error.type": type(exc).__name__}) + failed = True + if args.once: + telemetry.shutdown() + raise SystemExit(1 if failed else 0) + if idle(settings.JANITOR_INTERVAL_SECONDS): + telemetry.shutdown() + return + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 0000000..4f93308 --- /dev/null +++ b/main.py @@ -0,0 +1,666 @@ +import base64 +import binascii +import contextlib +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path +from typing import Literal +from urllib.parse import quote +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from opentelemetry import trace +from opentelemetry.trace import SpanKind +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.routing import Match + +import settings +import telemetry + +# lab6: телеметрия настраивается до первого соединения с БД (pool.open в lifespan). +telemetry.setup("messenger-api") + +import attachments # noqa: E402 +import auth # noqa: E402 +import db # noqa: E402 +import storage # noqa: E402 +import uploads # noqa: E402 +from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable # noqa: E402 + +log = logging.getLogger("messenger") + +# HTTP-метрики (contracts/observability.md). Метки: метод, ШАБЛОН маршрута, код и класс статуса — +# никаких реальных UUID, raw URL, request_id. Длительность — до отправки заголовков ответа +# (для скачивания файла тело потом стримится из S3 и в длительность не входит). +HTTP_DURATION = telemetry.meter.create_histogram( + "http.server.request.duration", unit="s", description="Duration of HTTP server requests") +HTTP_ACTIVE = telemetry.meter.create_up_down_counter( + "http.server.active_requests", unit="{request}", description="HTTP requests being processed now") +KNOWN_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + # Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3. + Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, " + "processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, + settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION, + extra={"event": "service.started"}) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + telemetry.shutdown() + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + # С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("attachment_ids must be unique") + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text cannot be empty without attachments") + return self + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] + + +class Variant(BaseModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +def route_template(request: Request) -> str: + """Шаблон маршрута (/api/v1/chats/{chat_id}/messages) — метка метрик и имя спана вместо raw URL.""" + for route in app.router.routes: + match, _ = route.matches(request.scope) + if match != Match.NONE: + return getattr(route, "path", "unmatched") + return "unmatched" + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + method = request.method if request.method in KNOWN_METHODS else "_OTHER" + route = route_template(request) + # Проверки здоровья (HAProxy раз в секунду, Docker) — только в метриках, без трасс: иначе это + # большая часть всех трасс. Входящий traceparent клиента продолжается (W3C Trace Context). + traced = not path.startswith("/health/") + span_scope = (telemetry.tracer.start_as_current_span( + f"{method} {route}", context=telemetry.extract(request.headers), kind=SpanKind.SERVER, + attributes={"http.request.method": method, "http.route": route, "url.scheme": "http", + "request.id": request_id}, + record_exception=False, set_status_on_exception=False) + if traced else contextlib.nullcontext(trace.INVALID_SPAN)) + HTTP_ACTIVE.add(1, {"http.request.method": method}) + status = 500 + try: + with span_scope as span: + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception as exc: + log.exception("Unhandled error", extra={"event": "http.unhandled_error", "request_id": request_id}) + telemetry.mark_error(span, type(exc).__name__) + response = error_response(request, "internal_error", "Internal server error", 500) + + status = response.status_code + span.set_attribute("http.response.status_code", status) + if status >= 500: + telemetry.mark_error(span, str(status)) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + trace_id = telemetry.trace_id_hex() if traced else None + if trace_id: + response.headers["X-Trace-Id"] = trace_id # расширение: найти трассу по ответу + elapsed = time.perf_counter() - started + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if not traced and status == 200 else logging.INFO + log.log(level, "%s %s -> %s (%.1f ms)", method, route, status, elapsed * 1000, + extra={"event": "http.request", "http.request.method": method, "http.route": route, + "http.response.status_code": status, "duration_ms": round(elapsed * 1000, 1), + "request_id": request_id}) + finally: + HTTP_ACTIVE.add(-1, {"http.request.method": method}) + elapsed = time.perf_counter() - started + HTTP_DURATION.record(elapsed, {"http.request.method": method, "http.route": route, + "http.response.status_code": status, + "http.response.status_class": f"{status // 100}xx"}) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + if cursor is None: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют. + + Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []} + равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними. + """ + body = data.model_dump(mode="json") + if not body.get("attachment_ids"): + body.pop("attachment_ids", None) + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=Attachment, + status_code=202, +) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str | None = Query(default=None), +): + """Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение + для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker. + + Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415) + -> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413), + проверка содержимого (415/413) и запись. + """ + user_id = current_user_id(request) + try: + await run_in_threadpool(db.check_member, chat_id, user_id) + name = uploads.validate_filename(filename) + declared = uploads.declared_type(request.headers.get("content-type")) + uploads.declared_length(request.headers.get("content-length")) + except ApiError: + await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке + raise + upload = await uploads.receive(request) + try: + image = await run_in_threadpool(inspect_traced, upload, declared) + row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image) + finally: + upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3) + return attachments.public(row) + + +def inspect_traced(upload, declared): + """Проверка содержимого — CPU в обработчике (не БД): в трассе видно отдельным спаном.""" + with telemetry.internal_span("inspect image", {"http.request.body.size": upload.size}): + return uploads.inspect(upload, declared) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public(db.get_attachment(attachment_id, current_user_id(request))) + + +VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"} + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + request: Request, + attachment_id: UUID, + variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"), +): + """Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3. + + queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404. + """ + row = db.get_attachment(attachment_id, current_user_id(request)) + if row["state"] != "ready": + raise conflict("Attachment is not ready") + if variant == "original": + key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"] + name = row["filename"] + else: + found = [v for v in row["variants"] if v["name"] == variant] + if not found: + raise not_found("Variant not found") + v = found[0] + key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"] + stem = row["filename"].rsplit(".", 1)[0] or "image" + name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}" + try: + stream = storage.open_read(key) + except storage.ObjectMissing: + log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"], + extra={"event": "attachment.object_missing", "attachment.id": str(row["id"])}) + raise unavailable("File is temporarily unavailable") from None + except storage.StorageUnavailable: + raise unavailable("File storage is temporarily unavailable") from None + headers = { + "Content-Length": str(size), + "Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + sha256 + '"', + } + return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + log_config=None, # логи uvicorn идут в общий JSON/OTLP-обработчик (telemetry.py) + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..e95f414 --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..8b981d0 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 0000000..73f0c84 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..5770e1d --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..35697dd --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,48 @@ +-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением. +-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать. + +-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата. +ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + -- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит. + object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'), + -- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна; + -- ready: объект в S3 сохранён, можно читать и прикреплять. + state text NOT NULL CHECK (state IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (width::bigint * height <= 40000000), + UNIQUE (id, chat_id), + -- владелец на момент загрузки был участником именно этого чата + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); + +-- janitor ищет старые незавершённые загрузки +CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + -- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id); + +-- С lab4 текст может быть пустым, если есть вложение (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/migrations/005_jobs.sql b/migrations/005_jobs.sql new file mode 100644 index 0000000..0751798 --- /dev/null +++ b/migrations/005_jobs.sql @@ -0,0 +1,133 @@ +-- lab5: асинхронная обработка вложений. Задания, outbox, варианты, журнал событий. +-- +-- Состояния attachments.state: +-- uploading — строка-намерение до записи в staging (как в lab4), клиенту не видна; +-- queued / processing / ready / failed — публичные состояния из контракта. +-- source_key — исходник в staging (staging/; у старых вложений lab4 — originals/); +-- object_key — опубликованный оригинал attachments//v/original, есть только у ready. + +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_check; +ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL; +ALTER TABLE attachments ADD COLUMN source_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer; + +-- Вложения lab4 были приняты синхронно и не имеют вариантов: переносим их в staging-модель и ставим +-- в очередь (ниже), worker опубликует оригинал в конечный prefix и сделает варианты. +UPDATE attachments SET source_key = object_key, object_key = NULL; +UPDATE attachments SET state = 'queued', updated_at = now() WHERE state = 'ready'; + +ALTER TABLE attachments ADD CONSTRAINT attachments_state_check + CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD CONSTRAINT attachments_source_key_check + CHECK (source_key ~ '^(staging|originals)/[a-f0-9]{32}$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_check + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/original$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code_check + CHECK (error_code ~ '^[a-z_]{1,64}$'); +-- ready <=> опубликованный оригинал; failed <=> код ошибки; до ready исходник обязан быть в staging. +ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object + CHECK ((state = 'ready') = (object_key IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_failed_has_code + CHECK ((state = 'failed') = (error_code IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_pending_has_source + CHECK (state NOT IN ('uploading', 'queued', 'processing') OR source_key IS NOT NULL); + +-- janitor: исходники ready/failed, которые пора удалить из staging +CREATE INDEX attachments_staging ON attachments (updated_at) + WHERE source_key IS NOT NULL AND state IN ('ready', 'failed'); + +-- Варианты. Первичный ключ = ключ эффекта (attachment_id, pipeline_version, variant): +-- повтор job не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/[a-z]+\.(jpg|png|webp)$'), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Задание: одна логическая работа на (вложение, версия pipeline). job_id не меняется при повторах. +-- attempt — управляемая попытка (claim worker'ом); lease_* — аренда processing с fencing-токеном. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts integer NOT NULL CHECK (max_attempts >= 1), + available_at timestamptz NOT NULL DEFAULT now(), + lease_token uuid, + lease_owner text, + lease_until timestamptz, + error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'), + last_error text CHECK (last_error ~ '^[a-z_]{1,64}$'), + replays integer NOT NULL DEFAULT 0, + traceparent text CHECK (traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + UNIQUE (attachment_id, pipeline_version), + CHECK (attempt <= max_attempts), + CHECK ((status = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)), + CHECK ((status = 'failed') = (error_code IS NOT NULL)) +); +CREATE INDEX attachment_jobs_leases ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_queued ON attachment_jobs (available_at) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_failed ON attachment_jobs (finished_at) WHERE status = 'failed'; + +-- Журнал (audit): claim, повторы, истечение аренды, дубли, replay. Только коды, без секретов и путей. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + event text NOT NULL CHECK (event ~ '^[a-z_]{1,64}$'), + detail text CHECK (char_length(detail) <= 200), + actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX job_events_by_job ON job_events (job_id, id); + +-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и задание. +-- dispatcher публикует строки с available_at <= now() и отмечает published_at после publisher confirm. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0 +); +CREATE INDEX outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_by_job ON outbox (job_id); + +-- Задания для вложений, принятых в lab4 (миграция = их producer). JOB_MAX_ATTEMPTS по умолчанию 5. +INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent) +SELECT gen_random_uuid(), id, 1, 'queued', 1, 5, + '00-' || md5(random()::text || id::text) || '-' || substr(md5(id::text), 1, 16) || '-01' +FROM attachments WHERE state = 'queued'; + +INSERT INTO job_events (job_id, attempt, event, detail, actor) +SELECT job_id, 1, 'created', 'lab4_backfill', 'migration 005' FROM attachment_jobs; + +INSERT INTO outbox (job_id, attempt, payload) +SELECT job_id, 1, jsonb_build_object( + 'schema_version', 1, + 'job_id', job_id, + 'type', 'attachment.process.v1', + 'attachment_id', attachment_id, + 'pipeline_version', pipeline_version, + 'attempt', 1, + 'created_at', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'traceparent', traceparent) +FROM attachment_jobs; diff --git a/ops.py b/ops.py new file mode 100644 index 0000000..d19a63d --- /dev/null +++ b/ops.py @@ -0,0 +1,201 @@ +"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher). + + docker compose exec dispatcher python ops.py status + docker compose exec dispatcher python ops.py failed [--limit N] + docker compose exec dispatcher python ops.py events + docker compose exec dispatcher python ops.py replay [--reason TEXT] + docker compose exec dispatcher python ops.py dlq [--limit N] + docker compose exec dispatcher python ops.py dlq-replay [--limit N] + +Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code, +число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя +обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit. + +replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready, +queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка +идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant). + +dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины): +failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу +ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения +остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3. +""" +import argparse +import json +import sys +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-ops") + +import broker # noqa: E402 +import pipeline # noqa: E402 + +ACTOR = "ops@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options="-c TimeZone=UTC") + + +def text(value): + if isinstance(value, (UUID,)): + return str(value) + if hasattr(value, "isoformat"): + return pipeline.utc_text(value) + return value + + +def rows_json(rows): + return [{key: text(value) for key, value in row.items()} for row in rows] + + +def cmd_status(conn, _args): + jobs = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()} + outbox = conn.execute( + """SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending, + count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due, + coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures, + extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL + AND available_at <= now())) AS oldest_due_s + FROM outbox""").fetchone() + leases = conn.execute( + """SELECT count(*) FILTER (WHERE lease_until >= now()) AS active, + count(*) FILTER (WHERE lease_until < now()) AS expired + FROM attachment_jobs WHERE status = 'processing'""").fetchone() + staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"] + report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v) + for k, v in outbox.items()}, + "leases": leases, "attachments_with_staging_source": staging} + try: + connection = broker.connect() + try: + report["broker"] = broker.check_topology(connection.channel()) + finally: + connection.close() + except broker.BrokerError as exc: + report["broker"] = {"error": type(exc).__name__} + return report + + +def cmd_failed(conn, args): + return rows_json(conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, + j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall()) + + +def cmd_events(conn, args): + return rows_json(conn.execute( + "SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id", + (args.job_id,)).fetchall()) + + +def cmd_replay(conn, args): + try: + return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)} + except pipeline.ReplayRefused as exc: + return {"refused": str(exc), "job_id": str(args.job_id)} + + +def dlq_messages(limit, handler): + """Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'.""" + connection = broker.connect() + results = [] + try: + channel = connection.channel() + keep = [] + for _ in range(limit): + method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False) + if method is None: + break + death = (properties.headers or {}).get("x-death") or [{}] + reason = death[0].get("reason") + reason = reason.decode() if isinstance(reason, bytes) else reason + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + message, problem = None, str(exc) + else: + problem = None + verdict, note = handler(message) + results.append({"job_id": message["job_id"] if message else None, "invalid": problem, + "dead_letter_reason": reason, "action": verdict, "note": note}) + if verdict == "ack": + channel.basic_ack(method.delivery_tag) + else: + keep.append(method.delivery_tag) + for tag in keep: # вернуть непрочитанными в DLQ + channel.basic_nack(tag, requeue=True) + finally: + connection.close() + return results + + +def cmd_dlq(_conn, args): + return dlq_messages(args.limit, lambda message: ("keep", "peek")) + + +def cmd_dlq_replay(conn, args): + def handle(message): + if message is None: + return "keep", "invalid message is kept for analysis" + job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s", + (UUID(message["job_id"]),)).fetchone() + if job is None: + return "keep", "unknown job" + if job["status"] == "failed": + try: + pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay") + return "ack", "replayed" + except pipeline.ReplayRefused as exc: + return "keep", str(exc) + if job["status"] == "ready": + return "ack", "already ready, no-op" + # queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново + conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s " + "AND status = 'queued'", (UUID(message["job_id"]),)) + return "ack", f"job is {job['status']}, handled by outbox/lease recovery" + return dlq_messages(args.limit, handle) + + +def main(): + parser = argparse.ArgumentParser(description="Attachment pipeline operator commands") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + p = sub.add_parser("failed") + p.add_argument("--limit", type=int, default=50) + p = sub.add_parser("events") + p.add_argument("job_id", type=UUID) + p = sub.add_parser("replay") + p.add_argument("job_id", type=UUID) + p.add_argument("--reason", default="operator replay") + p = sub.add_parser("dlq") + p.add_argument("--limit", type=int, default=20) + p = sub.add_parser("dlq-replay") + p.add_argument("--limit", type=int, default=100) + args = parser.parse_args() + if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()): + parser.error("--reason: up to 100 printable characters") + commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay, + "dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay} + # Действие оператора — отдельная трасса; replay ссылается (link) на трассу исходной загрузки. + with telemetry.internal_span(f"ops {args.command}", {"messenger.ops.command": args.command}): + with connect_db() as conn: + result = commands[args.command](conn, args) + telemetry.shutdown() + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..8cbe2cb --- /dev/null +++ b/pipeline.py @@ -0,0 +1,502 @@ +"""Задания обработки вложений: формат сообщения, outbox, переходы состояний. + +Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат) +и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции +вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT. + +Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать, +решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение +безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op. + +Переходы (contracts/attachment-states.md): + (нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202 + queued -> processing claim worker, аренда lease_until + fencing-токен lease_token + processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter) + recover_lease аренда истекла (worker умер/завис), попытка засчитана + processing -> ready finish_ready только при совпадении lease_token (fencing) + processing -> failed finish_failed постоянная ошибка или попытки исчерпаны + failed -> queued replay явная операция оператора (ops.py), с записью в журнал +""" +import json +import random +import re +import secrets +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from opentelemetry import trace +from opentelemetry.trace import SpanKind +from psycopg.types.json import Jsonb + +import settings +import telemetry + +SCHEMA_VERSION = 1 +JOB_TYPE = "attachment.process.v1" +EXCHANGE = "attachments" +ROUTING_KEY = "attachment.process.v1" +QUEUE = "attachments.process.v1" +DLX = "attachments.dlx" +DLQ = "attachments.process.v1.dlq" +DLQ_ROUTING_KEY = "attachment.process.v1.dlq" + +MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", + "created_at", "traceparent", "tracestate"} +REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"} +TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +CODE_RE = re.compile(r"^[a-z_]{1,64}$") + +# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES. +PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"} +FINAL_CODES = PERMANENT | {"retry_exhausted"} + + +class InvalidMessage(Exception): + """Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ.""" + + +def utc_text(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def new_traceparent() -> str: + """Запасной traceparent, если телеметрия выключена (OTEL_SDK_DISABLED): поле схемы остаётся заполненным.""" + return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01" + + +def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str | None, tracestate: str | None = None) -> dict: + """Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3. + + lab6: traceparent — контекст РЕАЛЬНОГО producer-спана, создавшего строку outbox (а не случайный ID). + Dispatcher при публикации заменяет его контекстом своего спана publish (тот же trace_id). + """ + message = { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": utc_text(datetime.now(timezone.utc)), + } + if traceparent: + message["traceparent"] = traceparent + if tracestate: + message["tracestate"] = tracestate[:512] + return message + + +def messaging_attributes(job_id: UUID, attachment_id: UUID, attempt: int, operation: str) -> dict: + """Атрибуты messaging-спанов (OTel semantic conventions) + технические ID задания.""" + return {"messaging.system": "rabbitmq", "messaging.operation.type": operation, + "messaging.destination.name": EXCHANGE, "messaging.rabbitmq.destination.routing_key": ROUTING_KEY, + "messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id), + "messenger.attachment.id": str(attachment_id), "messenger.job.attempt": attempt} + + +@contextmanager +def producer_span(job_id: UUID, attachment_id: UUID, attempt: int, parent_traceparent: str | None = None, + links: list | None = None): + """Спан «create attachments» (PRODUCER): строка outbox = сообщение, созданное в этой транзакции. + + Родитель — текущий спан (запрос API, попытка worker), а если его нет (фоновый dispatcher) — + parent_traceparent из строки задания. Возвращает (traceparent, tracestate) для сообщения. + """ + current = trace.get_current_span().get_span_context() + parent = None if current.is_valid else telemetry.context_from(parent_traceparent) + with telemetry.tracer.start_as_current_span( + f"create {EXCHANGE}", context=parent, kind=SpanKind.PRODUCER, links=links, + attributes=messaging_attributes(job_id, attachment_id, attempt, "create")): + carrier = telemetry.inject() + yield carrier.get("traceparent") or new_traceparent(), carrier.get("tracestate") + + +def parse_message(body: bytes) -> dict: + """Строгая проверка по схеме (additionalProperties: false, const, типы, форматы).""" + try: + message = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidMessage("not json") from None + if not isinstance(message, dict): + raise InvalidMessage("not an object") + if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message): + raise InvalidMessage("fields") + if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE: + raise InvalidMessage("schema_version/type") + for name in ("job_id", "attachment_id"): + if not isinstance(message[name], str) or not UUID_RE.match(message[name]): + raise InvalidMessage(name) + for name in ("pipeline_version", "attempt"): + value = message[name] + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidMessage(name) + try: + created = message["created_at"] + if not isinstance(created, str): + raise ValueError + datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError: + raise InvalidMessage("created_at") from None + if "traceparent" in message and (not isinstance(message["traceparent"], str) + or not TRACEPARENT_RE.match(message["traceparent"])): + raise InvalidMessage("traceparent") + if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512): + raise InvalidMessage("tracestate") + return message + + +def backoff_seconds(failed_attempt: int) -> float: + """Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна. + + Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS. + Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу). + """ + ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS, + settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1)) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None: + conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)", + (job_id, attempt, name, detail[:200] if detail else None, actor[:200])) + + +def _insert_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str, tracestate: str | None, delay_seconds: float) -> None: + payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) + conn.execute( + """INSERT INTO outbox (job_id, attempt, payload, available_at) + VALUES (%s, %s, %s, now() + make_interval(secs => %s))""", + (job_id, attempt, Jsonb(payload), delay_seconds), + ) + conn.execute("NOTIFY outbox") + + +def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + parent_traceparent: str | None, delay_seconds: float = 0, links: list | None = None) -> None: + """Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT.""" + with producer_span(job_id, attachment_id, attempt, parent_traceparent, links) as (traceparent, tracestate): + _insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate, + delay_seconds) + + +def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID: + """Новое задание для вложения, уже переведённого в queued (в транзакции API). + + attachment_jobs.traceparent — контекст исходной загрузки (спан create внутри запроса API): к нему + привязываются восстановление аренды и сверка, а replay оператора ссылается на него через link. + """ + job_id = uuid4() + with producer_span(job_id, attachment_id, 1) as (traceparent, tracestate): + conn.execute( + """INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, + traceparent) + VALUES (%s, %s, %s, 'queued', 1, %s, %s)""", + (job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent), + ) + event(conn, job_id, 1, "created", actor) + _insert_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent, tracestate, 0) + return job_id + + +# ---------- worker: захват и результат ---------- + +@dataclass +class Claim: + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + max_attempts: int + lease_token: UUID + source_key: str + content_type: str + size_bytes: int + sha256: str + traceparent: str | None + + +@dataclass +class Skip: + """Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker).""" + reason: str + + +@dataclass +class Reject: + """Сообщение не относится ни к одной job -> DLQ.""" + reason: str + + +def claim(conn, message: dict, owner: str, redelivered: bool): + """queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject.""" + job_id = UUID(message["job_id"]) + with conn.transaction(): + job = conn.execute( + """SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired + FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""", + (job_id,), + ).fetchone() + if job is None: + return Reject("unknown_job") + if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]: + return Reject("job_mismatch") + status = job["status"] + if status in ("ready", "failed"): + event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}") + return Skip(f"already_{status}") + attempt = job["attempt"] + if status == "queued": + if not job["due"]: + return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox + else: # processing + if not job["lease_expired"]: + event(conn, job_id, attempt, "duplicate_ignored", owner, "leased") + return Skip("leased") + # Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана. + event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker") + if attempt >= job["max_attempts"]: + _fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired") + return Skip("exhausted") + attempt += 1 + token = uuid4() + conn.execute( + """UPDATE attachment_jobs + SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, + lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id), + ) + att = conn.execute( + """UPDATE attachments SET state = 'processing', updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing') + RETURNING source_key, content_type, size_bytes, sha256""", + (job["attachment_id"],), + ).fetchone() + if att is None or att["source_key"] is None: + raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат + event(conn, job_id, attempt, "claimed", owner, + f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}") + return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token, + att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"]) + + +def renew_lease(conn, c: Claim) -> bool: + """Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить.""" + row = conn.execute( + """UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""", + (settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def _fenced(conn, c: Claim, owner: str) -> bool: + """Fencing: результат принимается, только если lease_token всё ещё наш.""" + row = conn.execute( + """SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing' + FOR UPDATE""", + (c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> float | None: + """processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT. + + Возвращает время от устойчивого приёма (COMMIT задания в API) до ready в секундах по часам + PostgreSQL (метрика messenger.attachment.time_to_ready) или None, если результат отклонён fencing. + """ + time_to_ready = None + with conn.transaction(): + if not _fenced(conn, c, owner): + stale = True + else: + stale = False + for v in variants: + conn.execute( + """INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""", + (c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"], + v["width"], v["height"], v["sha256"], v["object_key"]), + ) + # Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна). + updated = conn.execute( + """UPDATE attachments + SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now() + WHERE id = %s AND state = 'processing' + AND (pipeline_version IS NULL OR pipeline_version <= %s) + RETURNING id""", + (original_key, c.pipeline_version, c.attachment_id, c.pipeline_version), + ).fetchone() + if updated is None: + raise RuntimeError("attachment changed state during processing") + time_to_ready = conn.execute( + """UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s + RETURNING extract(epoch FROM clock_timestamp() - created_at)::float8 AS seconds""", + (c.job_id,), + ).fetchone()["seconds"] + event(conn, c.job_id, c.attempt, "ready", owner) + if stale: + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost") + return None + return time_to_ready + + +def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None): + conn.execute( + """UPDATE attachment_jobs + SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s""", + (code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id), + ) + conn.execute( + """UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing')""", + (code, attachment_id), + ) + event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}") + + +def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int, + traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None: + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (next_attempt, reason, delay, job_id), + ) + conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'", + (attachment_id,)) + add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay) + event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s") + + +def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str: + """Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'.""" + with conn.transaction(): + if not _fenced(conn, c, owner): + outcome = "stale" + elif permanent: + _fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None) + outcome = "failed" + elif c.attempt >= c.max_attempts: + _fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code) + outcome = "failed" + else: + _requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent, + backoff_seconds(c.attempt), code, owner, "retry_scheduled") + outcome = "retry" + if outcome == "stale": + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code) + return outcome + + +# ---------- dispatcher: аренды и сверка ---------- + +def recover_expired_leases(conn, actor: str, limit: int = 100) -> int: + """processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted).""" + recovered = 0 + with conn.transaction(): + rows = conn.execute( + """SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent + FROM attachment_jobs WHERE status = 'processing' AND lease_until < now() + ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""", + (limit,), + ).fetchall() + for job in rows: + # Спан в трассе исходной загрузки: видно, что попытку вернул dispatcher, а не worker. + with telemetry.tracer.start_as_current_span( + "recover expired lease", context=telemetry.context_from(job["traceparent"]), + attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}): + if job["attempt"] >= job["max_attempts"]: + _fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor, + "lease_expired") + else: + _requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], + job["attempt"] + 1, job["traceparent"], backoff_seconds(job["attempt"]), + "lease_expired", actor, "lease_expired") + recovered += 1 + return recovered + + +def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int: + """Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет. + + Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его + подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет. + """ + with conn.transaction(): + rows = conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent + FROM attachment_jobs j + WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s) + AND j.updated_at < now() - make_interval(secs => %s) + AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND + (o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s))) + ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""", + (older_than, older_than, older_than, limit), + ).fetchall() + for job in rows: + with telemetry.tracer.start_as_current_span( + "republish lost message", context=telemetry.context_from(job["traceparent"]), + attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}): + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], + job["traceparent"]) + conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],)) + event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight") + return len(rows) + + +# ---------- оператор: replay ---------- + +class ReplayRefused(Exception): + pass + + +def replay(conn, job_id: UUID, actor: str, reason: str) -> dict: + """failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются.""" + with conn.transaction(): + job = conn.execute( + """SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""", + (job_id,), + ).fetchone() + if job is None: + raise ReplayRefused("job not found") + if job["status"] != "failed": + # «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается. + raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs") + if job["source_key"] is None: + raise ReplayRefused("source file is no longer in staging (retention expired)") + attempt = job["attempt"] + 1 + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL, + available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now() + WHERE job_id = %s""", + (attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id), + ) + conn.execute( + "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'", + (job["attachment_id"],), + ) + event(conn, job_id, attempt, "replayed", actor, reason or "operator replay") + # Replay — новое решение оператора, а не продолжение исходного запроса: новая трасса со + # ссылкой (link) на исходную загрузку. Вызов из спана ops.py становится его родителем. + add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, None, + links=telemetry.link_to(job["traceparent"])) + return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6bf911f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,49 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +boto3==1.43.116 +botocore==1.43.116 +s3transfer==0.19.3 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 +pillow==12.3.0 +pika==1.3.2 +# lab6: OpenTelemetry SDK (traces/metrics/logs, OTLP/HTTP protobuf) + инструментирование psycopg +opentelemetry-api==1.45.2 +opentelemetry-sdk==1.45.2 +opentelemetry-exporter-otlp-proto-http==1.45.2 +opentelemetry-exporter-otlp-proto-common==1.45.2 +opentelemetry-proto==1.45.2 +opentelemetry-semantic-conventions==0.66b2 +opentelemetry-instrumentation==0.66b2 +opentelemetry-instrumentation-dbapi==0.66b2 +opentelemetry-instrumentation-psycopg==0.66b2 +opentelemetry-exporter-http-transport==0.66b2 +opentelemetry-exporter-otlp-common==0.66b2 +googleapis-common-protos==1.76.0 +protobuf==7.36.2 +requests==2.34.2 +certifi==2026.10.23 +charset-normalizer==3.5.2 +packaging==26.3 +wrapt==2.5.2 diff --git a/s3_init.py b/s3_init.py new file mode 100644 index 0000000..99489bc --- /dev/null +++ b/s3_init.py @@ -0,0 +1,103 @@ +"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init. + +Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает. +Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py): + admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init; + messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor. +Значения ключей не печатаются. +""" +import logging +import sys +import time +import urllib.error +import urllib.request +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +logging.basicConfig(level=logging.WARNING) + + +def client(key_id: str, secret: str): + return boto3.client( + "s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION, + aws_access_key_id=key_id, aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, + read_timeout=10, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + +def code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "?") + + +def anonymous_status(url: str, method: str = "GET") -> int: + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply: + return reply.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = settings.S3_BUCKET + admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"), + settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY) + + deadline = time.monotonic() + 90 + while True: + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if time.monotonic() > deadline: + print(f"s3-init: S3 is not ready: {type(exc).__name__}") + return 1 + time.sleep(1) + + try: + admin.create_bucket(Bucket=bucket) + print(f"s3-init: bucket {bucket} created") + except ClientError as exc: + if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + print(f"s3-init: cannot create bucket: {code(exc)}") + return 1 + print(f"s3-init: bucket {bucket} exists") + + # Ключ API работает со своим bucket ... + probe = "originals/s3-init-probe-" + uuid.uuid4().hex + api.put_object(Bucket=bucket, Key=probe, Body=b"probe") + api.get_object(Bucket=bucket, Key=probe)["Body"].read() + api.list_objects_v2(Bucket=bucket, MaxKeys=1) + api.delete_object(Bucket=bucket, Key=probe) + print("s3-init: api key can put/get/list/delete objects of its bucket") + # ... и не может администрировать хранилище. + intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8] + try: + api.create_bucket(Bucket=intruder) + admin.delete_bucket(Bucket=intruder) + print("s3-init: api key must NOT be able to create buckets") + return 1 + except ClientError as exc: + print(f"s3-init: api key cannot create buckets ({code(exc)})") + + # Приватность: анонимные запросы без подписи. + base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket + statuses = {"list": anonymous_status(base + "?list-type=2"), + "get": anonymous_status(base + "/originals/any"), + "put": anonymous_status(base + "/originals/any", "PUT")} + if any(status not in (401, 403) for status in statuses.values()): + print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}") + return 1 + print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_dashboard.py b/scripts/build_dashboard.py new file mode 100644 index 0000000..71b51c3 --- /dev/null +++ b/scripts/build_dashboard.py @@ -0,0 +1,265 @@ +"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git). + + python scripts/build_dashboard.py + +Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос +против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой. +Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти. +""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "infra/grafana/dashboards/messenger.json" + +PROM = {"type": "prometheus", "uid": "prometheus"} +LOKI = {"type": "loki", "uid": "loki"} +TEMPO = {"type": "tempo", "uid": "tempo"} + +API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO) +QUEUE = 'messaging_destination_name="attachments.process.v1"' + +_panels = [] +_next_id = [1] +_y = [0] + + +def _id(): + _next_id[0] += 1 + return _next_id[0] + + +def row(title): + _panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []}) + _y[0] += 1 + + +class Line: + """Раскладка панелей в строку сетки Grafana (24 колонки).""" + + def __init__(self, height): + self.x, self.h = 0, height + + def place(self, width): + pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]} + self.x += width + return pos + + def end(self): + _y[0] += self.h + + +def prom(expr, legend="", instant=False, fmt=None): + target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "", + "range": not instant, "instant": instant} + if fmt: + target["format"] = fmt + return target + + +def _with_refs(targets): + for n, target in enumerate(targets): + target["refId"] = chr(ord("A") + n) + return targets + + +def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None): + defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never", + "stacking": {"mode": "normal" if stack else "none"}}} + if minimum is not None: + defaults["min"] = minimum + if thresholds: + defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [ + {"color": color, "value": value} for value, color in thresholds]} + defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"} + _panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description, + "datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets), + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"}}}) + + +def stat(line, width, title, description, expr, unit, steps, decimals=None): + defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [ + {"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}} + if decimals is not None: + defaults["decimals"] = decimals + _panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM, + "gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]), + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "colorMode": "background", "graphMode": "none", "textMode": "value"}}) + + +def quantiles(metric, selector, by="le"): + return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))", + f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)] + + +def build(): + row("Обзор и SLO (окно SLO — 1 час)") + line = Line(4) + stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.", + "messenger:http_requests:rate1m", "reqps", [(None, "blue")]) + stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.", + "messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2) + stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).", + "messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")]) + stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.", + f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")]) + stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.", + "messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2) + stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.", + "messenger:slo_availability:error_budget_remaining_1h", "percentunit", + [(None, "red"), (0, "orange"), (0.5, "green")], 0) + stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.", + "messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2) + stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.", + "messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2) + line.end() + line = Line(5) + _panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)", + "description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.", + "datasource": PROM, "gridPos": line.place(24), + "targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]), + "transformations": [{"id": "organize", "options": {"excludeByName": { + "Time": True, "Value": True, "__name__": True}}}], + "fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}}) + line.end() + + row("API: запросы, ошибки, задержка (RED)") + line = Line(8) + timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.", + [prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", + "{{http_request_method}} {{http_route}}")], "reqps") + timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.", + [prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", + "{{http_response_status_class}}")], "reqps", stack=True) + timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.", + [prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) ' + f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")], + "percentunit", thresholds=[(0.05, "red")], minimum=0) + line.end() + line = Line(8) + timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.", + quantiles("http_server_request_duration_seconds", API), "s") + timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.", + [prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))", + "{{http_route}}")], "s") + timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.", + [prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short") + line.end() + + row("Фоновая обработка: очередь, worker, время до ready") + line = Line(8) + timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).", + [prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")], + "ops", stack=True) + timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.", + quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s") + timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).", + quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s", + thresholds=[(30, "red")]) + line.end() + line = Line(8) + timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.", + [prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"), + prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short") + timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.", + [prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"), + prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s", + thresholds=[(30, "red")]) + timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.", + [prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"), + prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")], + "ops") + line.end() + + row("Насыщение: пул БД, worker, CPU/RAM") + line = Line(8) + timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).", + [prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"), + prom("max(db_client_connection_max)", "max per replica"), + prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short") + timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).", + [prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))", + "p95 {{instance}}")], "s") + timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.", + [prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit", + thresholds=[(0.8, "orange"), (1, "red")], minimum=0) + line.end() + line = Line(8) + timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.", + [prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")], + "short") + timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.", + [prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)", + "{{job}} {{instance}}")], "percentunit") + timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).", + [prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"), + prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes") + line.end() + + row("Конвейер телеметрии: Collector и SDK") + line = Line(8) + timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.", + [prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"), + prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"), + prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"), + prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"), + prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")], + "ops") + timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.", + [prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"), + prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"), + prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"), + prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")], + "ops") + timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.", + [prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")], + "percentunit", minimum=0) + line.end() + line = Line(8) + timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.", + [prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes") + timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).", + [prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"), + prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"), + prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"), + prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")], + "ops") + line.end() + + row("Логи и трассы") + line = Line(9) + _panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)", + "description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.", + "datasource": LOKI, "gridPos": line.place(12), + "targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range", + "expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]), + "options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", + "enableLogDetails": True}}) + _panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)", + "description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.", + "datasource": TEMPO, "gridPos": line.place(12), + "targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql", + "query": "{ status = error }", "limit": 20, "tableType": "traces"}]), + "fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}}) + line.end() + + return { + "uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"], + "timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1, + "refresh": "10s", "time": {"from": "now-30m", "to": "now"}, + "templating": {"list": []}, "annotations": {"list": [{ + "builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]}, + "panels": _panels, + } + + +if __name__ == "__main__": + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels") diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..849974e --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,157 @@ +import base64 +import hashlib +import json +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore + + +def bucket_name() -> str: + """S3_BUCKET из .env (если задан), иначе значение по умолчанию.""" + env = ROOT / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("S3_BUCKET="): + return line.split("=", 1)[1].strip() + return "messenger-attachments" + + +def s3_config(bucket: str) -> str: + """Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket. + + Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects) + с суффиксом : действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет. + Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются. + """ + def read(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + config = {"identities": [ + {"name": "admin", + "credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ]} + return json.dumps(config, indent=2) + + +BROKER_VHOST = "messenger" +# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей). +EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1" +DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq" + + +def rabbit_hash(password: str) -> str: + """Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль)).""" + salt = secrets.token_bytes(4) + return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii") + + +def rabbit_definitions(passwords: dict) -> str: + """Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf). + + Права (configure / write / read, регулярные выражения по именам ресурсов): + dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет; + worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет. + Ни у кого из сервисов нет тега administrator и права создать/удалить очередь. + """ + queue_args = { + "x-queue-type": "quorum", + "x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ + "x-dead-letter-exchange": DLX, + "x-dead-letter-routing-key": DLQ_ROUTING_KEY, + "x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish) + "x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит + "x-max-length": 100000, + } + definitions = { + "users": [{"name": name, "password_hash": rabbit_hash(password), + "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} + for name, password in passwords.items()], + "vhosts": [{"name": BROKER_VHOST}], + "permissions": [ + {"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$", + "write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"}, + {"user": "worker", "vhost": BROKER_VHOST, "configure": "^$", + "write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"}, + ], + "exchanges": [ + {"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + {"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + ], + "queues": [ + {"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args}, + # DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный + # просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет. + {"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, + "arguments": {"x-queue-type": "classic"}}, + ], + "bindings": [ + {"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue", + "routing_key": ROUTING_KEY, "arguments": {}}, + {"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue", + "routing_key": DLQ_ROUTING_KEY, "arguments": {}}, + ], + } + return json.dumps(definitions, indent=2) + + +def broker_secrets(created: list, host: str = "rabbitmq") -> None: + if (SECRETS / "rabbitmq_definitions.json").exists(): + return + passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)} + for name, password in passwords.items(): + write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}") + write("rabbitmq_definitions.json", rabbit_definitions(passwords)) + created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"] + + +def main() -> None: + """Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes).""" + SECRETS.mkdir(exist_ok=True) + created = [] + if not (SECRETS / "postgres_password").exists(): + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + created += ["postgres_password", "database_url", "cache_url", "redis_auth"] + if not (SECRETS / "s3_root_password").exists(): + # admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init + write("s3_root_user", "root-" + secrets.token_hex(6)) + write("s3_root_password", secrets.token_hex(24)) + created += ["s3_root_user", "s3_root_password"] + if not (SECRETS / "s3_secret_access_key").exists(): + # отдельный ключ API с политикой только на свой bucket (создаёт s3-init) + write("s3_access_key_id", "api-" + secrets.token_hex(8)) + write("s3_secret_access_key", secrets.token_hex(24)) + created += ["s3_access_key_id", "s3_secret_access_key"] + if not (SECRETS / "s3_config.json").exists(): + write("s3_config.json", s3_config(bucket_name())) + created.append("s3_config.json") + broker_secrets(created) + if not (SECRETS / "grafana_admin_password").exists(): + # lab6: пароль администратора Grafana (GF_SECURITY_ADMIN_PASSWORD__FILE), логин admin + write("grafana_admin_password", secrets.token_urlsafe(18)) + created.append("grafana_admin_password") + if created: + print("Created in secrets/: " + ", ".join(created) + " (values are not printed).") + else: + print("secrets/ is complete, nothing changed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..3187e56 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,15 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, Prometheus, Loki, Tempo), then otel-collector, +rem api, janitor, dispatcher, worker, grafana, +rem then the balancer. Volumes are kept. +rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +cd /d "%~dp0.." +docker compose restart postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1 +docker compose restart otel-collector api janitor dispatcher worker grafana || exit /b 1 +docker compose restart lb || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps otel-collector api lb dispatcher worker grafana || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..08e5523 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ; lab6: Prometheus, Loki, Tempo), then +# otel-collector, api, janitor, dispatcher, +# worker, then the balancer. Volumes are kept (no down -v), nothing is re-seeded. +# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis s3 rabbitmq prometheus loki tempo +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq prometheus loki tempo +docker compose restart otel-collector api janitor dispatcher worker grafana +docker compose restart lb +docker compose up -d --wait --no-recreate --no-build --no-deps otel-collector api lb dispatcher worker grafana diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..8ffd0fa --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..cf4d724 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..5cdcce7 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/worker.cmd b/scripts/worker.cmd new file mode 100644 index 0000000..9ffab68 --- /dev/null +++ b/scripts/worker.cmd @@ -0,0 +1,8 @@ +@echo off +rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept). +cd /d "%~dp0.." +if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% ) +if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% ) +if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% ) +echo usage: %0 stop^|start^|status +exit /b 2 diff --git a/scripts/worker.sh b/scripts/worker.sh new file mode 100755 index 0000000..cab7c76 --- /dev/null +++ b/scripts/worker.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Demo hook for the defense: stop or start the worker without touching volumes. +# scripts/worker.sh stop | start | status +set -eu +cd "$(dirname "$0")/.." +case "${1:-}" in + stop) docker compose stop worker ;; + start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;; + status) docker compose ps worker ;; + *) echo "usage: $0 stop|start|status" >&2; exit 2 ;; +esac diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..e4cf3eb --- /dev/null +++ b/settings.py @@ -0,0 +1,111 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket. +S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333") +S3_REGION = env_str("S3_REGION", "us-east-1") +S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1) +S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5) +S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2) +S3_POOL_MAX = env_int("S3_POOL_MAX", 20) + +# Вложения: лимиты из контракта и защита памяти/времени. +MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) +DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2) +UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads") + +# Очистка незавершённых загрузок (janitor.py). +ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600) +JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + +# lab5: брокер (RabbitMQ) и фоновая обработка вложений. +# У API адреса брокера нет вообще: API пишет задание в outbox (PostgreSQL), публикует dispatcher. +BROKER_URL = read_secret("BROKER_URL_FILE", "BROKER_URL") +BROKER_HEARTBEAT_SECONDS = env_int("BROKER_HEARTBEAT_SECONDS", 30) +PIPELINE_VERSION = env_int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = env_int("WORKER_CONCURRENCY", 2) # потоков-потребителей в одном worker +WORKER_PREFETCH = env_int("WORKER_PREFETCH", 1) # неподтверждённых сообщений на поток +JOB_MAX_ATTEMPTS = env_int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = env_int("JOB_TIMEOUT_SECONDS", 60) # жёсткий предел одной обработки изображения +JOB_LEASE_SECONDS = env_int("JOB_LEASE_SECONDS", 15) # аренда processing, продлевается каждые lease/3 +JOB_BACKOFF_BASE_SECONDS = env_int("JOB_BACKOFF_BASE_SECONDS", 2) +JOB_BACKOFF_MAX_SECONDS = env_int("JOB_BACKOFF_MAX_SECONDS", 60) +JOB_RECONCILE_SECONDS = env_int("JOB_RECONCILE_SECONDS", 120) # queued без сообщения дольше -> опубликовать заново +WORKER_MEMORY_LIMIT_MB = env_int("WORKER_MEMORY_LIMIT_MB", 640) # RLIMIT_AS процесса обработки +WORKER_TMP_DIR = env_str("WORKER_TMP_DIR", "/tmp/work") +DISPATCHER_POLL_SECONDS = env_int("DISPATCHER_POLL_SECONDS", 1) +DISPATCHER_BATCH = env_int("DISPATCHER_BATCH", 100) +STAGING_RETENTION_SECONDS = env_int("STAGING_RETENTION_SECONDS", 86400) # исходник failed-вложения для replay +OUTBOX_RETENTION_SECONDS = env_int("OUTBOX_RETENTION_SECONDS", 7 * 86400) +ALIVE_FILE = env_str("ALIVE_FILE", "/tmp/alive") # healthcheck фоновых процессов + +# Учебное внедрение сбоев (faults.py). На стенде включено, в эксплуатации FAULT_INJECTION=0. +FAULT_INJECTION = env_str("FAULT_INJECTION", "0") == "1" +FAULTS_DIR = env_str("FAULTS_DIR", "/faults") + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY: + raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE") + if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS: + raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..b50ae35 --- /dev/null +++ b/storage.py @@ -0,0 +1,209 @@ +"""Приватное S3-хранилище (SeaweedFS) для байтов вложений. + +Раскладка ключей (lab5): + staging/<32 hex> — исходник, принятый API до ответа 202 (устойчивый staging); + attachments//v/original — опубликованный worker'ом оригинал (побайтно исходный файл); + attachments//v/thumbnail.jpg, crop.jpg, optimized.webp — варианты pipeline версии N; + originals/<32 hex> — исходники вложений lab4, worker обрабатывает их как staging. + +API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket +(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер, +имя файла пользователя в ключ не попадает. +""" +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings +import telemetry + +log = logging.getLogger("messenger.storage") + + +def _span(method: str, key: str | None = None): + """Клиентский спан S3: метод, bucket и ключ (ключи технические: staging/, attachments//...).""" + attributes = {"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": method, + "aws.s3.bucket": settings.S3_BUCKET} + if key is not None: + attributes["aws.s3.key"] = key + return telemetry.client_span(f"S3 {method}", attributes) + +ORIGINALS_PREFIX = "originals/" # lab4 +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "attachments/" +ALL_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, ORIGINALS_PREFIX) +READ_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей.""" + + +class ObjectMissing(Exception): + """Объекта нет в bucket (NoSuchKey / 404).""" + + +def _make_client(connect_timeout: float, read_timeout: float, attempts: int): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT, + region_name=settings.S3_REGION, + aws_access_key_id=settings.S3_ACCESS_KEY_ID or None, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # http://s3:8333// + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую + max_pool_connections=settings.S3_POOL_MAX, + # Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS) +# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor. +_quick_client = _make_client(1, 2, 1) + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "404", "NotFound") or status == 404 + + +def _unavailable(action: str, exc: Exception) -> StorageUnavailable: + detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else "" + log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail, + extra={"event": "s3.unavailable", "rpc.method": action, "error.type": type(exc).__name__}) + return StorageUnavailable() + + +def new_key(object_id: str) -> str: + """Ключ staging для нового исходника (имя файла пользователя в ключ не попадает).""" + return STAGING_PREFIX + object_id + + +def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + """Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект.""" + body.seek(0) + with _span("PutObject", key) as span: + span.set_attribute("http.request.body.size", size) + try: + _client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("put", exc) from None + + +def _get_body(key: str): + try: + return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"] + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing(key) from None + raise _unavailable("get", exc) from None + except BotoCoreError as exc: + raise _unavailable("get", exc) from None + + +def open_read(key: str): + """GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после. + + Спан покрывает запрос до получения заголовков; тело API стримит клиенту уже после. + """ + with _span("GetObject", key): + return _get_body(key) + + +def download(key: str, path, max_bytes: int) -> int: + """GET объекта в файл кусками. Дальше max_bytes не читаем: размер не совпадёт, worker отклонит исходник.""" + with _span("GetObject", key) as span: + stream = _get_body(key) + size = 0 + try: + with open(path, "wb") as file: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + break + size += len(chunk) + file.write(chunk) + if size > max_bytes: + break + except (BotoCoreError, OSError) as exc: + raise _unavailable("get", exc) from None + finally: + stream.close() + span.set_attribute("http.response.body.size", size) + return size + + +def iter_chunks(stream) -> Iterator[bytes]: + """Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание.""" + try: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return + yield chunk + except (BotoCoreError, OSError) as exc: + # Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length). + log.warning("S3 stream interrupted: %s", type(exc).__name__) + raise + finally: + stream.close() + + +def delete(key: str, quick: bool = False) -> None: + """Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа.""" + with _span("DeleteObject", key): + try: + (_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("delete", exc) from None + + +def exists(key: str) -> bool: + with _span("HeadObject", key): + try: + _client.head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise _unavailable("head", exc) from None + except BotoCoreError as exc: + raise _unavailable("head", exc) from None + + +def list_objects(prefix: str) -> Iterator[tuple[str, datetime, int]]: + """Все объекты под prefix: (key, last_modified, size). Постранично, по 1000.""" + paginator = _client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"], item["Size"] + except (BotoCoreError, ClientError) as exc: + raise _unavailable("list", exc) from None + + +def ping() -> None: + try: + _client.head_bucket(Bucket=settings.S3_BUCKET) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("head_bucket", exc) from None diff --git a/student-tests/check_configs.py b/student-tests/check_configs.py new file mode 100644 index 0000000..96441d0 --- /dev/null +++ b/student-tests/check_configs.py @@ -0,0 +1,107 @@ +"""Lab6: конфигурации наблюдаемости из Git проверены «родными» инструментами тех же версий + provisioning Grafana. + + python student-tests/check_configs.py + +В Docker-стенде (по умолчанию) инструменты запускаются в образах из compose.yaml: + otelcol-contrib validate, promtool check config/rules, promtool test rules (unit-тесты алертов), + loki -verify-config, tempo -config.verify; затем через HTTP API Grafana: источники данных созданы + provisioning-ом и отвечают (health), dashboard импортирован, его панели ссылаются на эти источники. +OBS_BIN_DIR=<каталог с бинарниками> — проверить конфиги локальными бинарниками без Docker (Grafana пропускается, +если GRAFANA_URL недоступен). +""" +import base64 +import json +import os +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + +from common import ROOT, check + +BIN = os.environ.get("OBS_BIN_DIR") +GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000").rstrip("/") + + +def run(title, docker_args, local_args): + if BIN: + command = [str(Path(BIN) / local_args[0]), *local_args[1:]] + else: + command = ["docker", "compose", *docker_args] + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + output = (result.stdout + result.stderr).strip() + tail = "\n".join(output.splitlines()[-6:]) + print(f"--- {title}: exit {result.returncode}\n{tail}") + check(result.returncode == 0, f"{title} failed") + + +def local_copy(path, replacements): + text = (ROOT / path).read_text(encoding="utf-8") + for old, new in replacements: + text = text.replace(old, new) + handle = tempfile.NamedTemporaryFile("w", suffix=Path(path).suffix, delete=False, encoding="utf-8") + handle.write(text) + handle.close() + return handle.name + + +def grafana(path): + password = (ROOT / "secrets/grafana_admin_password").read_text(encoding="utf-8").strip() + request = urllib.request.Request(GRAFANA_URL + path, headers={ + "Authorization": "Basic " + base64.b64encode(f"admin:{password}".encode()).decode(), "Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=10) as reply: + return json.loads(reply.read().decode("utf-8")) + + +def main(): + tmp = tempfile.mkdtemp() + run("otelcol-contrib validate (infra/otel/collector.yaml)", + ["run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"], + ["otelcol-contrib", "validate", f"--config={ROOT / 'infra/otel/collector.yaml'}"]) + run("promtool check config (infra/prometheus/prometheus.yml)", + ["exec", "-T", "prometheus", "promtool", "check", "config", "/etc/prometheus/prometheus.yml"], + ["promtool", "check", "config", local_copy("infra/prometheus/prometheus.yml", [ + ("/etc/prometheus/rules/", str(ROOT / "infra/prometheus/rules") + "/")])]) + run("promtool check rules (recording + alerts)", + ["exec", "-T", "prometheus", "sh", "-c", "promtool check rules /etc/prometheus/rules/*.yml"], + ["promtool", "check", "rules", *map(str, sorted((ROOT / "infra/prometheus/rules").glob("*.yml")))]) + run("promtool test rules (unit tests of alert thresholds/windows/for)", + ["exec", "-T", "prometheus", "promtool", "test", "rules", "/etc/prometheus/tests/alerts_test.yml"], + ["promtool", "test", "rules", str(ROOT / "infra/prometheus/tests/alerts_test.yml")]) + run("loki -verify-config (infra/loki/loki.yaml)", + ["run", "--rm", "--no-deps", "loki", "-config.file=/etc/loki/loki.yaml", "-verify-config"], + ["loki", f"-config.file={local_copy('infra/loki/loki.yaml', [('/loki', tmp + '/loki')])}", "-verify-config"]) + run("tempo -config.verify (infra/tempo/tempo.yaml)", + ["run", "--rm", "--no-deps", "tempo", "-config.file=/etc/tempo/tempo.yaml", "-config.verify=true"], + [os.environ.get("TEMPO_BIN", "tempo"), + f"-config.file={local_copy('infra/tempo/tempo.yaml', [('/var/tempo', tmp + '/tempo')])}", "-config.verify=true"]) + + try: + health = grafana("/api/health") + except (OSError, urllib.error.URLError) as exc: + check(BIN, f"Grafana is not reachable at {GRAFANA_URL}: {exc}") + print(f"--- Grafana: not reachable at {GRAFANA_URL}, skipped (local binaries mode)") + print("PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid") + return + print(f"--- Grafana {health.get('version')}: database {health.get('database')}") + sources = {d["uid"]: d for d in grafana("/api/datasources")} + for uid, kind in (("prometheus", "prometheus"), ("loki", "loki"), ("tempo", "tempo")): + check(uid in sources and sources[uid]["type"] == kind, f"datasource {uid} is not provisioned") + state = grafana(f"/api/datasources/uid/{uid}/health") + print(f"datasource {uid} ({kind}): {state.get('status')} {state.get('message', '')[:80]}") + check(state.get("status") == "OK", f"datasource {uid} is not healthy: {state}") + dashboard = grafana("/api/dashboards/uid/messenger-lab6") + panels = [p for p in dashboard["dashboard"]["panels"] if p["type"] != "row"] + used = {t.get("datasource", {}).get("uid") for p in panels for t in p.get("targets", [])} + print(f"dashboard '{dashboard['dashboard']['title']}' in folder '{dashboard['meta'].get('folderTitle')}': " + f"{len(panels)} panels, provisioned={dashboard['meta'].get('provisioned')}, datasources {sorted(used)}") + check(dashboard["meta"].get("provisioned"), "dashboard must come from provisioning, not the UI") + check(used <= set(sources), "panels reference a datasource that is not provisioned") + print("PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana " + "provisioned 3 healthy datasources and the dashboard without manual steps") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..eaea628 --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,119 @@ +"""Lab6: каждый запрос dashboard и правил выполняется против живых хранилищ. + + python student-tests/check_dashboard.py + +Проверяет, что dashboard ссылается на реально существующие метрики/метки (контракт наблюдаемости): + - каждое PromQL-выражение панелей и recording/alert rules выполняется без ошибки; + - для ключевых сигналов (RED API, задания, очередь, пул, worker, CPU/RAM) результат НЕ пуст; + - LogQL логов и TraceQL трасс выполняются; + - в метках метрик приложения нет запрещённых (user_id, chat_id, message_id, attachment_id, request_id, + url, filename, text) и нет значений-UUID (кроме instance). +Перед запуском нужен трафик: make test (smoke) — он создаёт запросы, задание и логи. +""" +import json +import re +import sys +from pathlib import Path + +import obs +from common import check + +ROOT = Path(__file__).resolve().parents[1] +DASHBOARD = ROOT / "infra/grafana/dashboards/messenger.json" +RULES = ROOT / "infra/prometheus/rules" +FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "url_full", + "filename", "text", "job_id", "trace_id", "span_id", "authorization"} +UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +# Метрики, которые обязаны иметь данные после smoke (имя -> зачем) +MUST_HAVE_DATA = { + 'http_server_request_duration_seconds_count{job="messenger-api"}': "HTTP requests/duration (3)", + "messenger_job_duration_seconds_count": "jobs processed/failed + duration (3)", + "messenger_attachment_time_to_ready_seconds_count": "end-to-end time to ready (4)", + "messenger_queue_depth": "queue depth (4)", + "messenger_queue_oldest_age_seconds": "oldest job age (4)", + "db_client_connection_count": "DB pool used/idle (4)", + "db_client_connection_pending_requests": "DB pool waiting (4)", + "messenger_worker_busy": "worker busy (4)", + "messenger_worker_capacity": "worker capacity (4)", + "process_cpu_time_seconds_total": "process CPU (4)", + "process_memory_usage_bytes": "process RAM (4)", + "otelcol_exporter_queue_size": "collector queue (4)", +} + + +def substitute(expr): + return expr.replace("$__rate_interval", "1m").replace("$__interval", "1m").replace("$__range", "1h") + + +def main(): + dashboard = json.loads(DASHBOARD.read_text(encoding="utf-8")) + prom_exprs, loki_exprs, tempo_exprs = [], [], [] + for panel in dashboard["panels"]: + for target in panel.get("targets", []): + uid = target.get("datasource", {}).get("uid") + if uid == "prometheus": + prom_exprs.append((panel["title"], target["expr"])) + elif uid == "loki": + loki_exprs.append((panel["title"], target["expr"])) + elif uid == "tempo": + tempo_exprs.append((panel["title"], target["query"])) + try: + import yaml # noqa: F401 — если PyYAML есть, проверим и выражения правил + rules = [] + for path in sorted(RULES.glob("*.yml")): + for group in yaml.safe_load(path.read_text(encoding="utf-8"))["groups"]: + for rule in group["rules"]: + rules.append((f"{path.name}: {rule.get('record') or rule.get('alert')}", rule["expr"])) + except ImportError: + rules = [] + + empty, errors = [], [] + for title, expr in prom_exprs + rules: + try: + rows = obs.prom_query(substitute(expr)) + except Exception as exc: # noqa: BLE001 + errors.append(f"{title}: {exc}") + continue + if not rows: + empty.append(title) + print(f"PromQL: {len(prom_exprs)} dashboard expressions + {len(rules)} rule expressions executed, " + f"{len(errors)} errors, {len(empty)} empty now (no traffic of that kind / Docker-only / no incident)") + for title in sorted(set(empty)): + print(f" empty now: {title}") + check(not errors, "PromQL errors:\n" + "\n".join(errors)) + + for title, expr in loki_exprs: + obs.loki_query(expr, minutes=60, limit=5) + print(f"LogQL ok: {title}: {expr}") + for title, query in tempo_exprs: + found = obs.tempo_search(query, minutes=60, limit=5) + print(f"TraceQL ok: {title}: {query} -> {len(found)} trace(s)") + + print() + for expr, why in MUST_HAVE_DATA.items(): + rows = obs.prom_query(expr) + check(rows, f"no data for {expr} ({why})") + print(f"data: {expr:<70} {len(rows):>3} series <- {why}") + + print() + bad = [] + for job in ("messenger-api", "messenger-worker", "messenger-dispatcher", "messenger-janitor"): + for series in obs.prom_series(f'{{job="{job}"}}'): + for name, value in series.items(): + if name in FORBIDDEN_LABELS: + bad.append(f"{series['__name__']}: forbidden label {name}") + if name != "instance" and UUID.search(value): + bad.append(f"{series['__name__']}: UUID value in label {name}") + labels = sorted({name for job in ("messenger-api", "messenger-worker", "messenger-dispatcher") + for series in obs.prom_series(f'{{job="{job}"}}') for name in series}) + print(f"labels used by application metrics: {', '.join(labels)}") + check(not bad, "high-cardinality labels:\n" + "\n".join(sorted(set(bad))[:20])) + route_values = sorted({row[0].get("http_route") for row in obs.prom_query( + 'count by (http_route) (http_server_request_duration_seconds_count{job="messenger-api"})')}) + print(f"http_route values (templates, not URLs): {route_values}") + check(all(not UUID.search(value or "") for value in route_values), "raw URL in http_route") + print("PASS: every dashboard/rule query runs, key signals have data, no forbidden labels") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 0000000..2cf933b --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,118 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + for one_shot in ("migrate", "s3-init"): + status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip() + check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis", "s3", "rabbitmq"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + if service in ("janitor", "worker", "dispatcher"): + # lab6: + внутренняя сеть telemetry (только до Collector), портов нет + check(not ports and networks == ["data", "telemetry"], f"{service}: networks data+telemetry, no ports") + if service == "otel-collector": + check(not ports and networks == ["telemetry"], "otel-collector: only the internal telemetry network, no ports") + if service in ("prometheus", "loki", "tempo", "grafana"): + check(ports and all(p.startswith("127.0.0.1:") for p in ports), f"{service}: ports only on 127.0.0.1") + check("data" not in networks and "edge" not in networks, f"{service} must not reach app data/edge networks") + check(api_count >= 2, "need at least 2 api replicas") + running = {r["Config"]["Labels"].get("com.docker.compose.service") for r in rows if r["State"]["Running"]} + check({"rabbitmq", "worker", "dispatcher"} <= running, "lab5: broker, worker and dispatcher must be running") + check(any(v.get("target") == "/var/lib/rabbitmq" and v.get("type") == "volume" + for v in services["rabbitmq"].get("volumes", [])), "rabbitmq needs a persistent volume") + check(services["rabbitmq"].get("hostname") == "rabbitmq", "rabbitmq needs a fixed hostname (node data dir)") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + # lab6: наблюдаемость + check(config["networks"]["telemetry"].get("internal") is True, "telemetry network must be internal") + check({"otel-collector", "prometheus", "loki", "tempo", "grafana"} <= running, + "lab6: collector, prometheus, loki, tempo and grafana must be running") + for name, target in (("prometheus", "/prometheus"), ("loki", "/loki"), ("tempo", "/var/tempo"), + ("grafana", "/var/lib/grafana")): + check(any(v.get("target") == target and v.get("type") == "volume" for v in services[name].get("volumes", [])), + f"{name} needs a persistent volume at {target}") + for name, service in services.items(): + image = service.get("image", "") + check(not image.endswith(":latest") and (":" in image or not image), f"{name}: image version must be pinned") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(not services["otel-collector"].get("secrets"), "otel-collector gets no secrets") + check({s["source"] for s in services["grafana"].get("secrets", [])} == {"grafana_admin_password"}, + "grafana: only its admin password") + for name in ("api", "worker", "dispatcher", "janitor", "prometheus", "loki", "tempo"): + check("grafana_admin_password" not in {s["source"] for s in services[name].get("secrets", [])}, + f"{name} must not get the Grafana password") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + for name in ("api", "janitor", "worker"): + names = {s["source"] for s in services[name]["secrets"]} + check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys") + check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key") + broker_secrets = {"broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions"} + for name in ("api", "janitor", "lb", "migrate"): + names = {s["source"] for s in services[name].get("secrets", [])} + check(not names & broker_secrets, f"{name} must not get broker credentials (API uses the outbox)") + check({s["source"] for s in services["dispatcher"]["secrets"]} == {"database_url", "broker_url_dispatcher"}, + "dispatcher: only database_url and its own broker user") + check(not {s["source"] for s in services["dispatcher"]["secrets"]} & {"s3_access_key_id", "s3_secret_access_key"}, + "dispatcher needs no S3 key") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, postgres/redis/s3/rabbitmq only in the internal data network, worker/dispatcher/janitor in " + f"data+telemetry, collector only in the internal telemetry network, prometheus/loki/tempo/grafana on " + f"127.0.0.1 with persistent volumes, pinned images, S3 admin keys only in s3 and s3-init, broker " + f"credentials only in dispatcher/worker") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 0000000..f57e649 --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,449 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") # по умолчанию evidence/ +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test) with telemetry ON, with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + # ---- lab6 ---- + ("configs", "lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning", + [PY, "student-tests/check_configs.py"]), + ("telemetry", "lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets", + [PY, "student-tests/test_telemetry.py", "request", "upload-chain", "retry-chain", "replay-link"]), + ("dashboard", "lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels", + [PY, "student-tests/check_dashboard.py"]), + ("telemetry-history", "lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes)", + [PY, "student-tests/test_telemetry.py", "history"]), + ("incident-worker", "lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved", + [PY, "student-tests/incidents.py", "worker-stop"]), + ("incident-db", "lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved", + [PY, "student-tests/incidents.py", "db-slow"]), + ("incident-telemetry", "lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted", + [PY, "student-tests/incidents.py", "telemetry-down"]), + ("incident-collector", "lab6: Collector down -> API unaffected, SDK queue bounded, drops counted", + [PY, "student-tests/incidents.py", "collector-down"]), + ("jobs-variants", "lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped", + [PY, "student-tests/test_jobs.py", "variants"]), + ("jobs-limits", "lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)", + [PY, "student-tests/test_jobs.py", "limits"]), + ("jobs-recovery", "lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits", + [PY, "student-tests/test_jobs.py", "worker-down", "broker-down"]), + ("jobs-outbox", "lab5: dispatcher crash between COMMIT and publish / after confirm", + [PY, "student-tests/test_jobs.py", "outbox-crash"]), + ("jobs-crash", "lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease", + [PY, "student-tests/test_jobs.py", "crash-after-put", "crash-after-commit", "staging", "lease"]), + ("jobs-retry", "lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ", + [PY, "student-tests/test_jobs.py", "retry", "exhausted-replay", "permanent", "dlq"]), + ("jobs-janitor-replay", "lab5: janitor does not delete objects re-written by replay (review fix)", + [PY, "student-tests/test_janitor_replay.py"]), + ("jobs-concurrency", "lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing", + [PY, "student-tests/test_jobs.py", "concurrency"]), + ("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids", + [PY, "student-tests/test_attachments.py"]), + ("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap", + [PY, "student-tests/test_s3_security.py"]), + ("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage", + [PY, "student-tests/test_consistency.py"]), + ("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas", + [PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]), + ("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)", + [PY, "student-tests/run_failover.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, " + f"janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: " + f"otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB.") + lines.append("") + lines += lab6_measurements() + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + memory = load_json("upload-memory.json") + if memory: + lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики " + f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). " + f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:") + lines.append("") + lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | " + "Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for run in memory["runs"]: + for n, r in enumerate(run["replicas"]): + head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | " + f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |" + cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-" + lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | " + f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |") + naive = memory["single_file_extra_rss_mib"] + lines.append("") + lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в " + f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка " + f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в " + f"{memory['api_limit_mib']} MiB.") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и " + f"прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + consistency = EVIDENCE / "consistency.txt" + if consistency.exists(): + text = consistency.read_text(encoding="utf-8") + timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text) + if len(timings) >= 2: + impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, " + f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | " + f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |") + impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за " + f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |") + if "PASS: S3 written but DB not committed" in text: + impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | " + "вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | " + "GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | " + "мусора не осталось; повторный проход ничего не делает |") + if "PASS: an interrupted upload leaves no row" in text: + impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | " + "строк и объектов не прибавилось, временных файлов 0 | мгновенно |") + impacts += lab5_impacts() + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def lab6_measurements(): + """Таблицы lab6 из evidence/incident-*.json (их пишет student-tests/incidents.py).""" + lines = [] + worker, db, tel, col = (load_json(f"incident-{n}.json") for n in ("worker-stop", "db-slow", "telemetry-down", + "collector-down")) + if worker or db or tel: + lines.append("Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):") + lines.append("") + lines.append("| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |") + lines.append("| --- | --- | --- | --- | --- |") + if worker: + for name, seconds in worker["firing_after_stop_s"].items(): + lines.append(f"| 1. worker | `docker compose stop worker` + 3 загрузки | {name} | {seconds} (от stop) | " + f"{worker['resolved_after_start_s'].get(name)} (от start) |") + if db: + lines.append(f"| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на {db['lock_seconds']} с под " + f"нагрузкой 8 потоков | MessengerHighErrorRate | {db['firing_after_lock_s']} (от LOCK) | " + f"{db['resolved_after_unlock_s']} (от COMMIT блокировки) |") + if tel: + lines.append(f"| 3. телеметрия | `docker compose stop {' '.join(tel['stopped'])}` на {tel['outage_s']} с | " + f"TelemetryExportFailing | {tel.get('TelemetryExportFailing_firing_at_s')} (от stop) | " + f"{tel['resolved_after_start_s']} (от start) |") + lines.append("") + if worker: + d = worker["diagnosis"] + lines.append(f"Инцидент 1, наблюдения в момент firing: consumers очереди = {d['queue_consumers']}, сообщений в " + f"RabbitMQ = {d['broker_queue_depth']}, queued заданий = {d['jobs_backlog_queued']}, возраст " + f"старейшего = {d['oldest_job_age_s']} с, возраст outbox = {d['outbox_oldest_age_s']} с (публикация " + f"работает), метрики worker не обновлялись {d['worker_metrics_last_seen_s']} с, последний лог worker: " + f"event=`{(d.get('last_worker_log') or {}).get('event')}`; трасса загрузки содержит `publish attachments`, " + f"но не содержит `process attachments.process.v1`. После start: вложения ready через " + f"{max(worker['ready_after_start_s'])} с, backlog {worker.get('after_recovery', {}).get('jobs_backlog_queued')}.") + lines.append("") + if db: + d, e, c = db["diagnosis"], db["diagnosis"]["error_trace"], db["diagnosis"]["handler_cpu_contrast"] + lines.append(f"Инцидент 2, наблюдения: доля 5xx (1m) = {d['http_5xx_ratio_1m']}, коды POST /messages за 1m = " + f"{d['status_codes_1m']}, p95 POST /messages = {d['http_p95_messages_s']} с, чтения других таблиц " + f"(GET /users/me, /chats) во время блокировки {d.get('reads_during_lock_ms')} мс, занято соединений " + f"пула = {d['pool_used']}, p95 ожидания пула = " + f"{d['pool_wait_p95_s']} с, логи API event=db.unavailable по error_type: " + f"{d['api_log_db_unavailable_by_error_type']}. Error-трасса {e['trace_id']}: server-спан " + f"{e['server_ms']} мс, из них `{e['slowest_db_span']}` {e['slowest_db_ms']} мс (доля " + f"{e['db_share']}, статус {e['db_span_status']}). Контраст «медленный обработчик» (трасса " + f"{c['trace_id']}, POST /users): server {c['server_ms']} мс, `argon2id hash` {c['argon2id_ms']} мс, " + f"все спаны БД {c['db_ms_total']} мс. Нагрузка: до {db['baseline']['success_rps']} успешных записей/с, " + f"во время блокировки {db['during_lock']['codes']}, после {db['after']['success_rps']}/с без ошибок.") + lines.append("") + if tel or col: + lines.append("| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |") + lines.append("| --- | --- | --- | --- |") + if tel: + probes = tel["probe_traces"] + lost = [p["sent_at_s"] for p in probes if not p["trace_in_tempo"]] + lines.append(f"| `stop {' '.join(tel['stopped'])}` на {tel['outage_s']} с (окно повторов Collector 120 с) | " + f"{tel['during']['requests']} запросов, коды {tel['during']['codes']}, доля успеха " + f"{tel['during']['success_ratio']} | heap Collector {tel['collector_heap_before_mib']} -> max " + f"{tel['collector_heap_max_mib']} MiB (limit 200), контейнер max {tel['collector_container_max_mib']} MiB | " + f"send/enqueue failed в Collector: {tel['lost_in_collector']}; пробные трассы дошли " + f"{len(probes) - len(lost)}/{len(probes)}, потеряны отправленные на {lost} с |") + if col: + lines.append(f"| `stop otel-collector` на {col['outage_s']} с | {col['during']['requests']} запросов, коды " + f"{col['during']['codes']}, доля успеха {col['during']['success_ratio']} | api {col['api_mib_before']} " + f"-> max {col['api_mib_max']} MiB (2 реплики) | SDK api потерял спанов: {col['sdk_lost_spans_api']} " + f"(queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start " + f"экспорт возобновился ({col['sdk_exported_ok_after_restart']} спанов за минуту) |") + lines.append("") + return lines + + +def lab5_line(name, pattern): + """Строка вывода test_jobs.py из evidence/.txt, содержащая pattern (или None).""" + path = EVIDENCE / f"{name}.txt" + if not path.exists(): + return None + for line in path.read_text(encoding="utf-8").splitlines(): + if pattern in line and not line.startswith(("#", "---")): + return line.strip() + return None + + +def lab5_impacts(): + """Строки таблицы отказов lab5: наблюдение — дословно строка вывода теста (числа из прогона).""" + rows = [ + ("jobs-recovery", "worker started", "worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker", + "202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready", + "потерь нет; staging удалён после COMMIT"), + ("jobs-recovery", "rabbitmq started", "rabbitmq остановлен во время upload", + "202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется", "потерь нет"), + ("jobs-outbox", "killed between COMMIT and publish", "crash dispatcher между COMMIT задания и publish", + "строка outbox не отмечена -> публикуется после рестарта процесса", "без ручного действия"), + ("jobs-outbox", "after publisher confirm", "crash dispatcher после confirm, до отметки outbox", + "дубль сообщения, worker распознаёт по состоянию job", "один набор вариантов"), + ("jobs-crash", "killed after S3 PUT", "crash worker после PUT в S3, до COMMIT и ack", + "повтор после истечения аренды перезаписывает те же ключи", "один набор, 4 объекта"), + ("jobs-crash", "after COMMIT, before ack", "crash worker после COMMIT, до ack", + "redelivery ready-задачи -> no-op ack", "очередь пуста, дублей нет"), + ("jobs-crash", "frozen attempt", "worker завис (не продлевает аренду)", + "dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing", "ready не откатан"), + ("jobs-limits", "imaging hang", "зависание обработки изображения", + "процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась", "повтор -> ready"), + ("jobs-limits", "memory bomb", "нехватка памяти в процессе обработки (RLIMIT_AS)", + "failed/resource_limit, worker жив", "постоянная ошибка, без повторов"), + ("jobs-retry", "storage_unavailable ->", "2 временные ошибки S3", "backoff с jitter, затем ready", "-"), + ("jobs-retry", "retry_exhausted (public", "5 временных ошибок подряд", + "failed/retry_exhausted, реестр failed, content 409", "replay оператора"), + ("jobs-retry", "ops replay", "replay failed и повторный replay ready", "тот же job_id -> ready; повтор отклонён", + "дубля нет"), + ("jobs-retry", "-> reject -> DLQ", "невалидное сообщение и неизвестный job_id", "reject -> DLQ, dlq-replay не исполняет", + "сообщения сохранены"), + ("jobs-concurrency", "peak processing", "8 заданий в очереди, старт worker", + "WORKER_CONCURRENCY=2, prefetch 1", "-"), + ] + out = [] + for name, pattern, what, expected, recovery in rows: + line = lab5_line(name, pattern) + if line: + out.append(f"| {what} | {expected} | {line.replace('|', '/')} | {recovery} |") + return out + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed and os.environ.get("REPORT_UPDATE", "1") == "1": + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..66cc2a8 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,347 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None): + """POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся.""" + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body=body, headers={"Idempotency-Key": key or new_key()}) + + +def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=202, timeout=60, + wait=True): + """POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра. + + Lab5: успешный приём — 202 со снимком queued (переход 201 -> 202 из COURSE.md). При wait=True + ответ дополняется ожиданием готовности: .json — метаданные после GET в состоянии ready/failed, + .accepted — исходный снимок 202. Так проверки lab4 («файл скачивается побайтно») сохраняются. + """ + query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="") + headers = {"Content-Type": content_type} if content_type is not None else {} + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data, + token=user["token"] if user else None, headers=headers, timeout=timeout) + if reply.status == 202: + check(reply.json["status"] == "queued" and reply.json["variants"] == [], "202 must be a queued snapshot") + reply.accepted = reply.json + if wait: + reply.json = wait_attachment(user, reply.json["id"]) + return reply + + +def wait_attachment(user, attachment_id, statuses=("ready", "failed"), seconds=60): + """GET метаданных, пока вложение не перейдёт в одно из statuses (smoke ждёт ready до 60 с).""" + deadline = time.monotonic() + seconds + while True: + att = call("GET", f"/api/v1/attachments/{attachment_id}", 200, token=user["token"]).json + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"attachment {attachment_id} is {att['status']} after {seconds} s") + time.sleep(0.2) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def app_python(code, *args, service="api", env=None): + """Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API). + + Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON). + """ + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return output + + +def janitor(*args, env=None): + """Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт.""" + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += ["janitor", "python", "janitor.py", "--once", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return json.loads(output.splitlines()[-1]) + + +def sql_value(query): + """Одно значение из PostgreSQL (psql -A -t).""" + code, output, error = psql(query) + check(code == 0, f"psql failed: {error[-300:]}") + return output + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 0000000..8f97f1c --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..81416de --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 0000000..01cd9e7 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,76 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY m.created_at, m.id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 0000000..19421a1 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38faa2bd4a2bb00b572233684dd5474d46572d38 GIT binary patch literal 1081 zcmex=^lOiET&UP@Y7ModgWM?qOlT~kX_QeM|USHnP6LsJ7}2qQZ?I~NC+Fc+7w zhLo6;2Fc+6AqGJXh5&{NW=16jCP7AKLB{__7$kv?WCpnw4wxC3SlHNsQWA_{0GD83 zU}0ipXa0YSfrpuqfk}W_0O-e0Us7g@q%7gR_iLh?m&0PUC4!SCXq;RjxhlxgB!%5l zB;^Y4y)P5pd>t06DhN&v(m3fMxhj9ZT2I4sr=BT|yYB~Rob&`LVloP1l{8g0_4@wg z;slu|fs=#Gt1kgXlzOH#fHkY6t=#u#mJ#!uE~6l}_q85C?}^ISER2f9~O z0L335cMAd?%K;Q+WM%509st@UCI}20q>y1?U}9hdTLuhFdqzD7I4%wW$=wjJ@hAk` zcmx4&%0bQs8iisK%-KvJlU88?Kusb7=z0pEUS+dq62l%C5cdHC!(Oo`S>l)>t%&2W z6vDs^^q0V|se8Fk9r(1>j{TMMuhu{Ow;I1Ktz-Vh1QNe>2U zW@aX!`+*(+s;&jfGq4D<3Mm>ovIz$!vMUve7&T5@$f4}C@t|nX#SbdRNkvVZTw>x9 zl2WQ_>Kd9_CZ=ZQ7M51dF0O9w9-dyoA)#U65s^{JDXD4c8JStdC8cHM6_r)ZEv;?s z9i3g1CQq3GGAU*RJ2V zdF$b$$4{OPfBE|D`;VW$K>lK6V1{@LNJ2b@<}X2@znFm0!om*n7b88f2KE_o9%~}Y zXK;@p{B?_ghnW!=dCYp^Nk zI`iL#Ki&>f`#Y>_;hTV?BFa#mKfu=BySiva)fccMR;=xKyLZJ}+l8;3MHfC61R6dA zsN_%Cy{lsJIJc*Fss0-!LA1S;hP4?4;WTe zNi6EY^g|NJ;Qc@!feePYdTl7gV2F?4&Tusb`z-gZYYxzbu2CzhCLGPE06H8TNU&hp z7a3yT2M$D#1O7sTygm!&S4fcV2U`gY#9&*Xv# bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: + """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). + + pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах. + idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). + """ + channels = 1 if gray else 3 + header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace) + if idat is None: + compressor = zlib.compressobj(6) + parts = [] + # Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход. + passes = [(width, height)] if not interlace else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + for pass_width, pass_height in passes: + if pass_width <= 0 or pass_height <= 0: + continue + row = b"\x00" + bytes(pass_width * channels) + for _ in range(pass_height): + parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row)) + parts.append(compressor.flush()) + idat = b"".join(parts) + body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) + if pad_to is not None: + need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0" + assert need >= 0, "image is already larger than pad_to" + body += chunk(b"tEXt", b"Comment\x00" + b"a" * need) + return body + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями).""" + side = int(((target_bytes - 4096) / 3) ** 0.5) + return png(side, side, noise=True) + + +def sample_png() -> bytes: + return (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (FIXTURES / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (FIXTURES / "progressive.jpg").read_bytes() + + +def flip_byte(data: bytes, position: int) -> bytes: + return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:] + + +GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + + +# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) --- + +def ihdr(width, height, depth=8, color=2, interlace=0): + return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + + +def zero_idat(width, height, depth=8, channels=3): + """Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный).""" + row_bytes = (width * depth * channels + 7) // 8 + return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height) + + +def assemble(*chunks): + """PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан.""" + if not chunks or chunks[-1][0] != b"IEND": + chunks = chunks + ((b"IEND", b""),) + return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks) diff --git a/student-tests/incidents.py b/student-tests/incidents.py new file mode 100644 index 0000000..b89f607 --- /dev/null +++ b/student-tests/incidents.py @@ -0,0 +1,499 @@ +"""Lab6, уровень 4: контролируемые инциденты, алерты firing -> resolved и разбор причины по телеметрии. + + python student-tests/incidents.py worker-stop # (1) остановлен worker: растёт возраст задач + python student-tests/incidents.py db-slow # (2) PostgreSQL медленный (блокировка таблицы) + python student-tests/incidents.py telemetry-down # (3) недоступны Tempo и Loki + python student-tests/incidents.py collector-down # (3') недоступен сам Collector + python student-tests/incidents.py # все по очереди (~20 минут) + +Каждый сценарий: нагрузка/загрузки -> отказ -> ожидание firing в Prometheus -> «диагностика» только по +сохранённым наблюдениям (метрики, логи, трассы) -> восстановление -> ожидание resolved. Хронология +печатается, итог пишется в evidence/incident-<имя>.json (без токенов и текстов). Тест сам возвращает +стенд в рабочее состояние (start/unlock), даже если упал по проверке. +""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid + +import obs +from common import ROOT, call, check, compose, new_chat, new_user, try_call, wait_attachment +from images import png + +EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") +FIRE_TIMEOUT = 300 +RESOLVE_TIMEOUT = 300 +T0 = time.monotonic() + + +def say(text): + print(f"[{time.monotonic() - T0:6.1f}s] {text}", flush=True) + + +def save(name, data): + EVIDENCE.mkdir(exist_ok=True) + path = EVIDENCE / f"incident-{name}.json" + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + say(f"saved {path}") + + +def q(expr, default=None): + try: + value = obs.prom_value(expr, default) + except Exception: # noqa: BLE001 — Prometheus может быть недоступен в сценарии + return default + return default if value is None or value != value else value # NaN (нет данных) -> default + + +def rnd(value, digits=3): + return None if value is None else round(value, digits) + + +class Load: + """Closed-loop POST сообщений в N потоков. Хранит (время, код или None, задержка) — без тела ответов.""" + + def __init__(self, threads=4, think=0.05, timeout=12): + self.user = new_user("incident") + self.chat = new_chat(self.user, title="Incident load") + self.threads, self.think, self.timeout = threads, think, timeout + self.samples = [] + self.lock = threading.Lock() + self.stop_event = threading.Event() + self.workers = [] + + def _run(self): + while not self.stop_event.is_set(): + started = time.monotonic() + reply = try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", body={"text": "load"}, + token=self.user["token"], headers={"Idempotency-Key": uuid.uuid4().hex}, + timeout=self.timeout) + with self.lock: + self.samples.append((started, reply.status if reply else None, time.monotonic() - started)) + self.stop_event.wait(self.think) + + def start(self): + for _ in range(self.threads): + thread = threading.Thread(target=self._run, daemon=True) + thread.start() + self.workers.append(thread) + return self + + def stop(self): + self.stop_event.set() + for thread in self.workers: + thread.join(timeout=self.timeout + 2) + + def window(self, since, until=None): + until = until or time.monotonic() + with self.lock: + rows = [s for s in self.samples if since <= s[0] < until] + codes = {} + for _, status, _ in rows: + key = str(status) if status else "timeout/conn" + codes[key] = codes.get(key, 0) + 1 + ok = [lat for _, status, lat in rows if status and status < 500] + ok.sort() + p95 = ok[int(len(ok) * 0.95) - 1] if ok else None + seconds = max(until - since, 0.001) + return {"requests": len(rows), "codes": codes, "success_ratio": rnd(len(ok) / len(rows), 4) if rows else None, + "success_rps": rnd(len(ok) / seconds, 1), "p95_ms_successful": rnd(p95 * 1000 if p95 else None, 1)} + + +def container_mem_mib(service): + """Память контейнера (docker stats) — работает и когда Prometheus/Collector недоступны.""" + ids = subprocess.run(["docker", "compose", "ps", "-q", service], cwd=ROOT, capture_output=True, + text=True).stdout.split() + total = 0.0 + for cid in ids: + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}", cid], cwd=ROOT, + capture_output=True, text=True).stdout.strip() + if not out: + continue + data = json.loads(out.splitlines()[0]) + if "MemUsageBytes" in data: + total += data["MemUsageBytes"] / 2**20 + continue + used = data.get("MemUsage", "0B").split("/")[0].strip() + for unit, factor in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("kB", 1 / 1024), ("MB", 1), ("GB", 1024), + ("B", 1 / 2**20)): + if used.endswith(unit): + total += float(used[:-len(unit)]) * factor + break + return round(total, 1) + + +def traced_upload(user, chat_id, size=(64, 48)): + trace_id, traceparent = obs.new_traceparent() + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename=incident.png", 202, raw=png(*size), + token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent}) + return reply.json["id"], trace_id + + +def probe(user, chat_id): + """Запрос с известным trace_id: после восстановления проверим, дошла ли его трасса до Tempo.""" + trace_id, traceparent = obs.new_traceparent() + reply = try_call("POST", f"/api/v1/chats/{chat_id}/messages", body={"text": "probe"}, token=user["token"], + headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}, timeout=10) + return trace_id, reply.status if reply else None + + +def counter(expr): + return q(f"sum({expr})", 0.0) or 0.0 + + +# --------------------------------------------------------------------------------------------------- +def worker_stop(): + """(1) Остановлен worker. Ожидаем: MessengerWorkerNoConsumers и MessengerQueueLagGrowing -> firing; + причина находится по метрикам очереди, логам и обрыву трасс; после start backlog -> 0, resolved.""" + result = {"scenario": "worker-stop", "date": time.strftime("%Y-%m-%d %H:%M:%S")} + user = new_user("lag") + chat = new_chat(user, title="Lag chat") + check(obs.alerts().get("MessengerQueueLagGrowing") is None, "MessengerQueueLagGrowing is already active") + say("baseline: consumers=%s, oldest job age=%s s" % ( + q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'), + q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'))) + stopped = time.monotonic() + compose("stop", "worker") + say("docker compose stop worker") + try: + uploads = [traced_upload(user, chat["id"]) for _ in range(3)] + say(f"uploaded {len(uploads)} attachments: 202 queued (API accepts without worker)") + fired = {} + deadline = time.monotonic() + FIRE_TIMEOUT + while len(fired) < 2: + states = obs.alerts() + for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"): + if states.get(name) == "firing" and name not in fired: + fired[name] = round(time.monotonic() - stopped, 1) + say(f"ALERT {name} firing ({fired[name]} s after stop)") + if len(fired) < 2: + say(" consumers=%s backlog queued=%s oldest age=%s s, alerts: %s" % ( + q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'), + q('max(messenger_jobs_backlog{status="queued"})'), + rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1), + {k: v for k, v in states.items() if k.startswith("Messenger")})) + check(time.monotonic() < deadline, f"alerts not firing after {FIRE_TIMEOUT} s: {fired}") + time.sleep(5) + result["firing_after_stop_s"] = fired + + # Диагностика только по сохранённым наблюдениям: + diag = { + "queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'), + "broker_queue_depth": q('max(messenger_queue_depth{messaging_destination_name="attachments.process.v1"})'), + "jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'), + "oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1), + "outbox_oldest_age_s": rnd(q("max(messenger_outbox_oldest_age_seconds)"), 1), + "published_ok_last_5m": q('sum(increase(messenger_messages_published_total{outcome="ok"}[5m]))'), + "worker_metrics_last_seen_s": rnd(q('time() - max(timestamp(messenger_worker_capacity))'), 0), + } + last_worker_log = obs.loki_query('{service_name="messenger-worker"}', minutes=15, limit=1) + diag["last_worker_log"] = ({"event": last_worker_log[0][0].get("event"), "message": last_worker_log[0][1]} + if last_worker_log else None) + spans = obs.tempo_trace(uploads[0][1]) + names = sorted({s["name"] for s in spans}) + diag["upload_trace_span_names"] = names + diag["upload_trace_has_consumer_span"] = any(s["name"].startswith("process ") for s in spans) + result["diagnosis"] = diag + say("diagnosis: " + json.dumps(diag, ensure_ascii=False)) + check(diag["queue_consumers"] == 0, "consumers must be 0 while worker is stopped") + check("publish attachments" in names and not diag["upload_trace_has_consumer_span"], + "trace must end at publish: message is in the broker, nobody consumes it") + finally: + started = time.monotonic() + compose("start", "worker") + say("docker compose start worker") + ready_at = [] + for attachment_id, _ in uploads: + att = wait_attachment(user, attachment_id, seconds=120) + check(att["status"] == "ready", f"{attachment_id} is {att['status']}") + ready_at.append(round(time.monotonic() - started, 1)) + result["ready_after_start_s"] = ready_at + say(f"all attachments ready {max(ready_at)} s after worker start; backlog now %s" % + q('max(messenger_jobs_backlog{status="queued"})')) + resolved = {} + for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"): + obs.wait_alert(name, "resolved", RESOLVE_TIMEOUT, log=say) + resolved[name] = round(time.monotonic() - started, 1) + result["resolved_after_start_s"] = resolved + result["after_recovery"] = { + "jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'), + "oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1), + "queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})')} + say(f"alerts resolved: {result['resolved_after_start_s']}; now {result['after_recovery']}") + result["time_to_ready_p95_5m_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))'), 1) + save("worker-stop", result) + print("PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; " + "start -> backlog drained -> resolved") + + +# --------------------------------------------------------------------------------------------------- +LOCK_SECONDS = 210 # > окно rate (1m) + for (1m) + задержка экспорта метрик с запасом + + +def db_slow(): + """(2) PostgreSQL «медленный»: таблица messages заблокирована (ACCESS EXCLUSIVE) на LOCK_SECONDS. + Ожидаем MessengerHighErrorRate firing; трассы показывают, что время уходит в INSERT (PostgreSQL), + а не в обработчик; после снятия блокировки — resolved.""" + result = {"scenario": "db-slow", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "lock_seconds": LOCK_SECONDS} + load = Load(threads=8, think=0.05).start() + say("load: 8 threads POST /messages") + time.sleep(25) + base_from = time.monotonic() - 20 + result["baseline"] = load.window(base_from) + result["baseline"]["server_p95_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 4) + say(f"baseline: {result['baseline']}") + check(obs.alerts().get("MessengerHighErrorRate") is None, "MessengerHighErrorRate is already active") + + lock_out = {} + + def hold_lock(): + command = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", + "-q", "-c", f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({LOCK_SECONDS}); COMMIT;"] + lock_out["code"] = subprocess.run(command, cwd=ROOT, capture_output=True).returncode + lock_out["released"] = time.monotonic() # COMMIT выполнен: блокировка снята + + locker = threading.Thread(target=hold_lock, daemon=True) + locked = time.monotonic() + locker.start() + say(f"LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for {LOCK_SECONDS} s (PostgreSQL alive, writes wait)") + try: + fired_at = None + while fired_at is None: + if obs.alerts().get("MessengerHighErrorRate") == "firing": + fired_at = round(time.monotonic() - locked, 1) + break + say(" 5xx ratio=%s p95=%s s pool used=%s waiting=%s" % ( + rnd(q("messenger:http_error_ratio:rate1m"), 3), + rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'), 3), + q('sum(db_client_connection_count{db_client_connection_state="used"})'), + q("sum(db_client_connection_pending_requests)"))) + check(time.monotonic() - locked < LOCK_SECONDS + 30, "MessengerHighErrorRate did not fire") + time.sleep(5) + say(f"ALERT MessengerHighErrorRate firing ({fired_at} s after lock)") + result["firing_after_lock_s"] = fired_at + + # Диагностика: метрики + diag = { + "http_5xx_ratio_1m": rnd(q("messenger:http_error_ratio:rate1m"), 3), + "http_p95_messages_s": rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 3), + "pool_used": q('sum(db_client_connection_count{db_client_connection_state="used"})'), + "pool_wait_p95_s": rnd(q("histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))"), 3), + "status_codes_1m": {row[0].get("http_response_status_code"): rnd(row[1], 2) for row in obs.prom_query( + 'sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m]))')}, + } + # Чтения других таблиц во время блокировки messages: быстрые -> «медленно» не всё, а одна операция + reads = [] + for _ in range(5): + began = time.monotonic() + call("GET", "/api/v1/users/me", 200, token=load.user["token"]) + call("GET", "/api/v1/chats", 200, token=load.user["token"]) + reads.append((time.monotonic() - began) * 1000 / 2) + diag["reads_during_lock_ms"] = {"GET /users/me + GET /chats, avg per request": rnd(sum(reads) / len(reads), 1), + "max": rnd(max(reads), 1)} + # Логи: какие ошибки БД видит API + logs = obs.loki_query('{service_name="messenger-api"} | event="db.unavailable"', minutes=5, limit=500) + by_type = {} + for labels, _ in logs: + by_type[labels.get("error_type", "?")] = by_type.get(labels.get("error_type", "?"), 0) + 1 + diag["api_log_db_unavailable_by_error_type"] = by_type + # Трассы: медленный запрос с ошибкой — где время + found = obs.tempo_search('{ resource.service.name = "messenger-api" && name = "POST /api/v1/chats/{chat_id}/messages" && status = error }', + minutes=5, limit=5) + check(found, "no error traces of POST messages in Tempo") + spans = obs.wait_trace(found[0]["traceID"].rjust(32, "0"), seconds=30) + server = next(s for s in spans if s["kind"] == "SERVER") + db_spans = [s for s in spans if s["kind"] == "CLIENT" and s["name"] in ("INSERT", "SELECT", "UPDATE")] + slowest = max(db_spans, key=lambda s: s["duration_ms"]) + diag["error_trace"] = {"trace_id": found[0]["traceID"], "server_ms": rnd(server["duration_ms"], 1), + "slowest_db_span": slowest["name"], "slowest_db_ms": rnd(slowest["duration_ms"], 1), + "db_span_status": slowest["status"], "db_share": rnd(slowest["duration_ms"] / server["duration_ms"], 3), + "tree": obs.span_tree(spans).splitlines()} + # Контраст: медленный ОБРАБОТЧИК (CPU Argon2id при регистрации) — время в своём спане, БД быстрая + trace_id, traceparent = obs.new_traceparent() + call("POST", "/api/v1/users", 201, body={"username": f"cpu_{uuid.uuid4().hex[:10]}", "display_name": "Cpu", + "password": "Test-password-123"}, headers={"traceparent": traceparent}) + cpu = obs.wait_trace(trace_id, lambda s: any(x["name"] == "argon2id hash" for x in s), 30) + cpu_server = next(s for s in cpu if s["kind"] == "SERVER") + argon = next(s for s in cpu if s["name"] == "argon2id hash") + cpu_db = [s for s in cpu if s["kind"] == "CLIENT"] + diag["handler_cpu_contrast"] = {"trace_id": trace_id, "server_ms": rnd(cpu_server["duration_ms"], 1), + "argon2id_ms": rnd(argon["duration_ms"], 1), + "db_ms_total": rnd(sum(s["duration_ms"] for s in cpu_db), 1), + "tree": obs.span_tree(cpu).splitlines()} + result["diagnosis"] = diag + say("diagnosis: " + json.dumps({k: v for k, v in diag.items() if k not in ("error_trace", "handler_cpu_contrast")}, + ensure_ascii=False)) + say("error trace (DB): " + json.dumps({k: v for k, v in diag["error_trace"].items() if k != "tree"})) + print("\n".join(" " + line for line in diag["error_trace"]["tree"])) + say("contrast, slow handler (CPU): " + json.dumps({k: v for k, v in diag["handler_cpu_contrast"].items() if k != "tree"})) + print("\n".join(" " + line for line in diag["handler_cpu_contrast"]["tree"])) + check(diag["error_trace"]["db_share"] > 0.8, "in the DB incident most of the request time must be in a DB span") + check(diag["handler_cpu_contrast"]["argon2id_ms"] > diag["handler_cpu_contrast"]["db_ms_total"], + "in the CPU contrast the handler span must dominate the DB spans") + finally: + locker.join(timeout=LOCK_SECONDS + 30) + unlocked = lock_out.get("released", time.monotonic()) + say(f"lock released (psql exit {lock_out.get('code')}), {round(unlocked - locked, 1)} s after LOCK") + result["during_lock"] = load.window(locked + 5, unlocked) + obs.wait_alert("MessengerHighErrorRate", "resolved", RESOLVE_TIMEOUT, log=say) + result["resolved_after_unlock_s"] = round(time.monotonic() - unlocked, 1) + say(f"ALERT MessengerHighErrorRate resolved {result['resolved_after_unlock_s']} s after unlock") + result["after"] = load.window(unlocked + 5) + load.stop() + say(f"load: during lock {result['during_lock']}; after {result['after']}") + check((result["after"]["success_ratio"] or 0) > 0.99, "writes must recover after the lock") + save("db-slow", result) + print("PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span " + "(not the handler); unlock -> resolved") + + +# --------------------------------------------------------------------------------------------------- +def telemetry_down(stores=("tempo", "loki"), outage=150): + """(3) Хранилища трасс и логов недоступны дольше окна повторов Collector (120 с). + Ожидаем: API работает без ошибок, RSS Collector ограничен, TelemetryExportFailing firing, + потери считаются (send_failed) и видны на пробных трассах; после start — данные снова идут.""" + result = {"scenario": "telemetry-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "stopped": list(stores), + "outage_s": outage} + load = Load(threads=4, think=0.05).start() + time.sleep(20) + result["baseline"] = load.window(time.monotonic() - 15) + failed_before = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}") + for sig in ("spans", "log_records")} + heap_before = q("max(otelcol_process_runtime_heap_alloc_bytes)") + mem_before = container_mem_mib("otel-collector") + result["collector_heap_before_mib"] = rnd((heap_before or 0) / 2**20, 1) + result["collector_container_before_mib"] = mem_before + say(f"baseline: {result['baseline']}, collector heap {result['collector_heap_before_mib']} MiB, " + f"container {mem_before} MiB") + compose("stop", *stores) + stopped = time.monotonic() # отсчёт после остановки (Loki при stop сбрасывает данные на диск ~десятки секунд) + say(f"docker compose stop {' '.join(stores)}: stopped") + samples, probes = [], [] + try: + while time.monotonic() - stopped < outage: + probes.append((round(time.monotonic() - stopped, 1), *probe(load.user, load.chat["id"]))) + # memory_limiter ограничивает heap Go-процесса (RSS ещё включает отображённый в память бинарник ~150 MiB) + sample = {"t": round(time.monotonic() - stopped, 1), + "collector_heap_mib": rnd((q("max(otelcol_process_runtime_heap_alloc_bytes)") or 0) / 2**20, 1), + "collector_rss_mib": rnd((q("max(otelcol_process_memory_rss)") or 0) / 2**20, 1), + "collector_container_mib": container_mem_mib("otel-collector"), + "queue": {r[0].get("exporter"): r[1] for r in obs.prom_query("otelcol_exporter_queue_size")}, + "alert": obs.alerts().get("TelemetryExportFailing", "inactive")} + samples.append(sample) + say(f" {sample}") + time.sleep(10) + finally: + compose("start", *stores) + started = time.monotonic() + say(f"docker compose start {' '.join(stores)}") + result["during"] = load.window(stopped, started) + result["samples"] = samples + result["collector_heap_max_mib"] = max(s["collector_heap_mib"] or 0 for s in samples) + result["collector_rss_max_mib"] = max(s["collector_rss_mib"] or 0 for s in samples) + result["collector_container_max_mib"] = max(s["collector_container_mib"] or 0 for s in samples) + fired = [s["t"] for s in samples if s["alert"] == "firing"] + result["TelemetryExportFailing_firing_at_s"] = fired[0] if fired else None + # после восстановления: Collector досылает очередь; затем проверяем, какие пробные трассы дошли + obs.wait_metric("sum(otelcol_exporter_queue_size)", lambda v: v == 0, 180) + result["queue_drained_after_start_s"] = round(time.monotonic() - started, 1) + say(f"exporter queues drained {result['queue_drained_after_start_s']} s after start") + time.sleep(15) # Tempo/Loki индексируют принятые пакеты + found = [] + for t, trace_id, status in probes: + spans = [] + for _ in range(5): + spans = obs.tempo_trace(trace_id) + if spans: + break + time.sleep(2) + found.append({"sent_at_s": t, "http": status, "trace_in_tempo": bool(spans)}) + result["probe_traces"] = found + failed_after = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}") + for sig in ("spans", "log_records")} + result["lost_in_collector"] = {sig: round(failed_after[sig] - failed_before[sig]) for sig in failed_after} + obs.wait_alert("TelemetryExportFailing", "resolved", RESOLVE_TIMEOUT, log=say) + result["resolved_after_start_s"] = round(time.monotonic() - started, 1) + result["after"] = load.window(started + 5) + load.stop() + say(f"API during outage: {result['during']}") + say(f"probe traces: {sum(p['trace_in_tempo'] for p in found)}/{len(found)} reached Tempo " + f"(lost: sent at {[p['sent_at_s'] for p in found if not p['trace_in_tempo']]} s)") + say(f"lost in Collector (send_failed+enqueue_failed): {result['lost_in_collector']}; collector heap " + f"{result['collector_heap_before_mib']} -> max {result['collector_heap_max_mib']} MiB (memory_limiter 200 MiB), " + f"container max {result['collector_container_max_mib']} MiB") + check(result["during"]["success_ratio"] == 1.0, "API must keep working while telemetry stores are down") + check(result["collector_heap_max_mib"] < 200, "collector heap must stay below memory_limiter limit_mib") + save("telemetry-down", result) + print("PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved") + + +def collector_down(outage=90): + """(3') Сам Collector недоступен: SDK в процессах приложения ограничивает очередь и отбрасывает лишнее.""" + result = {"scenario": "collector-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "outage_s": outage} + load = Load(threads=4, think=0.05).start() + time.sleep(20) + result["baseline"] = load.window(time.monotonic() - 15) + # Потери SDK: отброшено процессором (очередь полна) + не принято exporter-ом (ошибка соединения). + sdk_lost = ('otel_sdk_processor_span_processed_total{job="messenger-api", error_type!=""}', + 'otel_sdk_exporter_span_exported_total{job="messenger-api", error_type!=""}') + lost_before = [counter(expr) for expr in sdk_lost] + api_before = container_mem_mib("api") + say(f"baseline: {result['baseline']}, api containers {api_before} MiB") + stopped = time.monotonic() + compose("stop", "otel-collector") + say("docker compose stop otel-collector") + samples = [] + try: + while time.monotonic() - stopped < outage: + samples.append({"t": round(time.monotonic() - stopped, 1), "api_mib": container_mem_mib("api")}) + say(f" {samples[-1]}") + time.sleep(10) + finally: + compose("start", "otel-collector") + started = time.monotonic() + say("docker compose start otel-collector") + result["during"] = load.window(stopped, started) + result["api_mib_before"] = api_before + result["api_mib_max"] = max(s["api_mib"] for s in samples) + time.sleep(30) # SDK досылает cumulative-метрики, включая счётчик отброшенных спанов + lost_after = [counter(expr) for expr in sdk_lost] + result["sdk_lost_spans_api"] = {"processor_queue_full": round(lost_after[0] - lost_before[0]), + "exporter_failed": round(lost_after[1] - lost_before[1])} + result["sdk_exported_ok_after_restart"] = round(q( + 'sum(increase(otel_sdk_exporter_span_exported_total{job="messenger-api", error_type=""}[1m]))', 0) or 0) + result["after"] = load.window(started + 5) + load.stop() + say(f"API during collector outage: {result['during']}; api memory {api_before} -> max {result['api_mib_max']} MiB; " + f"SDK lost spans (api): {result['sdk_lost_spans_api']}; exported OK in the last minute after restart: " + f"{result['sdk_exported_ok_after_restart']}") + check(sum(result["sdk_lost_spans_api"].values()) > 0, "losses during the outage must be counted by the SDK") + check(result["sdk_exported_ok_after_restart"] > 0, "spans must flow again after the Collector is back") + check(result["during"]["success_ratio"] == 1.0, "API must keep working without the Collector") + check(result["api_mib_max"] - api_before < 100, "API memory must not grow without bound") + save("collector-down", result) + print("PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start") + + +SCENARIOS = {"worker-stop": worker_stop, "db-slow": db_slow, "telemetry-down": telemetry_down, + "collector-down": collector_down} + +if __name__ == "__main__": + names = sys.argv[1:] or list(SCENARIOS) + unknown = [n for n in names if n not in SCENARIOS] + if unknown: + raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}") + failed = 0 + for name in names: + print(f"=== {name}", flush=True) + try: + SCENARIOS[name]() + except Exception as exc: # noqa: BLE001 + failed += 1 + print(f"FAIL: {name}: {exc}", flush=True) + for service in ("worker", "otel-collector", "tempo", "loki"): + compose("start", service) + print(f"{len(names) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..998548c --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/obs.py b/student-tests/obs.py new file mode 100644 index 0000000..4410e2c --- /dev/null +++ b/student-tests/obs.py @@ -0,0 +1,223 @@ +"""Запросы к хранилищам телеметрии для тестов lab6 (только стандартная библиотека). + +Адреса — порты, опубликованные compose.yaml только на 127.0.0.1: + PROMETHEUS_URL (http://127.0.0.1:9090), LOKI_URL (http://127.0.0.1:3100), TEMPO_URL (http://127.0.0.1:3200). +""" +import base64 +import binascii +import json +import os +import re +import secrets +import time +import urllib.error +import urllib.parse +import urllib.request + +PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090").rstrip("/") +LOKI_URL = os.environ.get("LOKI_URL", "http://127.0.0.1:3100").rstrip("/") +TEMPO_URL = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200").rstrip("/") + + +def get_json(url, timeout=10): + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=timeout) as reply: + return json.loads(reply.read().decode("utf-8")) + + +# ---------- Prometheus ---------- + +def prom_query(expr, at=None): + """Мгновенный запрос. Возвращает список (labels, float). Ошибка запроса -> исключение.""" + params = {"query": expr} + if at is not None: + params["time"] = str(at) + data = get_json(f"{PROMETHEUS_URL}/api/v1/query?" + urllib.parse.urlencode(params)) + if data.get("status") != "success": + raise RuntimeError(f"PromQL error: {data}") + result = data["data"]["result"] + if data["data"]["resultType"] == "scalar": + return [({}, float(result[1]))] + return [(item["metric"], float(item["value"][1])) for item in result] + + +def prom_value(expr, default=None): + rows = prom_query(expr) + return rows[0][1] if rows else default + + +def prom_series(match): + data = get_json(f"{PROMETHEUS_URL}/api/v1/series?" + urllib.parse.urlencode({"match[]": match})) + return data["data"] + + +def alerts(): + """Активные алерты: {имя: состояние (pending/firing)}.""" + data = get_json(f"{PROMETHEUS_URL}/api/v1/alerts") + out = {} + for alert in data["data"]["alerts"]: + name = alert["labels"]["alertname"] + if out.get(name) != "firing": + out[name] = alert["state"] + return out + + +def wait_alert(name, state, seconds, poll=5, log=print): + """Ждёт, пока алерт name перейдёт в state ('firing' или 'resolved' = отсутствует). Возвращает секунды.""" + started = time.monotonic() + while True: + current = alerts().get(name, "inactive") + done = current == "firing" if state == "firing" else current == "inactive" + if done: + return round(time.monotonic() - started, 1) + if time.monotonic() - started > seconds: + raise AssertionError(f"alert {name}: still {current} after {seconds} s (want {state})") + log(f" ... {name}: {current} ({time.monotonic() - started:.0f} s)") + time.sleep(poll) + + +def wait_metric(expr, predicate, seconds, poll=5): + """Ждёт, пока значение выражения удовлетворит predicate. Возвращает значение.""" + deadline = time.monotonic() + seconds + while True: + value = prom_value(expr) + if value is not None and predicate(value): + return value + if time.monotonic() > deadline: + raise AssertionError(f"{expr}: value {value} after {seconds} s") + time.sleep(poll) + + +# ---------- Loki ---------- + +def loki_query(logql, minutes=30, limit=200): + """Записи за последние minutes минут: список (labels+structured metadata, строка).""" + now = time.time_ns() + params = {"query": logql, "start": str(now - minutes * 60 * 10**9), "end": str(now), "limit": str(limit), + "direction": "backward"} + data = get_json(f"{LOKI_URL}/loki/api/v1/query_range?" + urllib.parse.urlencode(params)) + if data.get("status") != "success": + raise RuntimeError(f"LogQL error: {data}") + out = [] + for stream in data["data"]["result"]: + for value in stream["values"]: + labels = dict(stream["stream"]) + if len(value) > 2 and isinstance(value[2], dict): + for group in value[2].values(): + if isinstance(group, dict): + labels.update(group) + out.append((labels, value[1])) + return out + + +def wait_logs(logql, minimum=1, seconds=60): + deadline = time.monotonic() + seconds + while True: + rows = loki_query(logql) + if len(rows) >= minimum: + return rows + if time.monotonic() > deadline: + raise AssertionError(f"Loki: {len(rows)} records for {logql} after {seconds} s") + time.sleep(2) + + +# ---------- Tempo ---------- + +def new_traceparent(): + """Свой trace_id для контролируемого запроса: по нему трасса находится без поиска.""" + trace_id = secrets.token_hex(16) + return trace_id, f"00-{trace_id}-{secrets.token_hex(8)}-01" + + +def tempo_trace(trace_id): + """Спаны трассы: список dict(service, name, kind, span_id, parent_id, duration_ms, status, attrs, links).""" + try: + data = get_json(f"{TEMPO_URL}/api/traces/{trace_id}") + except urllib.error.HTTPError as error: + if error.code == 404: + return [] + raise + spans = [] + for batch in data.get("batches", data.get("resourceSpans", [])): + resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])} + for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])): + for span in scope.get("spans", []): + spans.append({ + "service": resource.get("service.name"), + "instance": resource.get("service.instance.id"), + "name": span["name"], + "kind": span.get("kind", "").replace("SPAN_KIND_", ""), + "span_id": _hex_id(span["spanId"], 16), + "parent_id": _hex_id(span.get("parentSpanId", ""), 16), + "start": int(span["startTimeUnixNano"]), + "duration_ms": (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6, + "status": span.get("status", {}).get("code", "").replace("STATUS_CODE_", ""), + "attrs": {a["key"]: _value(a["value"]) for a in span.get("attributes", [])}, + "links": [{"trace_id": _hex_id(link.get("traceId", ""), 32), + "span_id": _hex_id(link.get("spanId", ""), 16)} for link in span.get("links", [])], + }) + return spans + + +def _hex_id(value, size): + """ID спана/трассы из JSON Tempo: hex или base64 (OTLP JSON кодирует bytes в base64) -> hex.""" + if not value: + return "" + if re.fullmatch(r"[0-9a-fA-F]+", value) and len(value) in (size, size // 2 * 2): + return value.lower().rjust(size, "0") + try: + return base64.b64decode(value).hex().rjust(size, "0") + except (binascii.Error, ValueError): + return value + + +def _value(value): + for key in ("stringValue", "intValue", "boolValue", "doubleValue"): + if key in value: + return int(value[key]) if key == "intValue" else value[key] + return json.dumps(value) + + +def wait_trace(trace_id, predicate=lambda spans: bool(spans), seconds=60): + deadline = time.monotonic() + seconds + while True: + spans = tempo_trace(trace_id) + if predicate(spans): + return spans + if time.monotonic() > deadline: + raise AssertionError(f"Tempo: trace {trace_id} incomplete after {seconds} s ({len(spans)} spans)") + time.sleep(2) + + +def tempo_search(traceql, minutes=30, limit=20): + now = int(time.time()) + params = {"q": traceql, "start": str(now - minutes * 60), "end": str(now), "limit": str(limit)} + return get_json(f"{TEMPO_URL}/api/search?" + urllib.parse.urlencode(params)).get("traces", []) + + +def span_tree(spans): + """Текстовое дерево трассы для отчёта (без атрибутов с данными пользователя).""" + by_id = {s["span_id"]: s for s in spans} + children = {} + for span in spans: + parent = span["parent_id"] if span["parent_id"] in by_id else "" + children.setdefault(parent, []).append(span) + lines = [] + + def walk(span, depth): + extra = [] + for key in ("messenger.job.attempt", "messenger.job.outcome", "http.response.status_code", "error.type", + "aws.s3.key"): + if key in span["attrs"]: + extra.append(f"{key}={span['attrs'][key]}") + status = " ERROR" if span["status"] == "ERROR" else "" + links = f" links={len(span['links'])}" if span["links"] else "" + short = (span["service"] or "?").replace("messenger-", "") + lines.append(f"{' ' * depth}- [{short}] {span['name']} ({span['kind']}, {span['duration_ms']:.1f} ms)" + f"{status}{links} {' '.join(extra)}".rstrip()) + for child in sorted(children.get(span["span_id"], []), key=lambda s: s["start"]): + walk(child, depth + 1) + + for root in sorted(children.get("", []), key=lambda s: s["start"]): + walk(root, 0) + return "\n".join(lines) diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 0000000..6535eae --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 0000000..f2f3ffb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 0000000..0f28ec8 --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 0000000..80c555d --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,51 @@ +import os +import re +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + match = re.match(r"^.+?-lab\d+-(.+)-\d+$", name) + restarted.append(match.group(1) if match else name) + services = ("postgres", "redis", "s3", "rabbitmq", "api", "janitor", "dispatcher", "worker", "lb", + "otel-collector", "prometheus", "loki", "tempo", "grafana") + for service in services: + check(service in restarted, f"{service} was not restarted by the hook") + print(f"PASS: the hook restarted {', '.join(services)} (all api replicas; same containers, new start time); " + f"data, the attachment bytes and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..7c770d9 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,258 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_empty_cursor, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..e57fe28 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,338 @@ +"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ. + +Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа: +пользовательская ошибка не должна становиться 500. +""" +import hashlib +import http.client +import socket +import struct +import urllib.parse +import uuid +import zlib + +import images +from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload + +URL = urllib.parse.urlsplit(BASE_URL) + + +def setup(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + return alice, bob, eve, new_chat(alice, bob) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def check_error(reply, code): + check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}") + + +def test_png_and_jpeg_roundtrip(): + alice, bob, _, chat = setup() + for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"), + (images.sample_jpeg(), "image/jpeg", "sample.jpg"), + (images.progressive_jpeg(), "image/jpeg", "progressive.jpg"), + (images.png(33, 17, interlace=1), "image/png", "interlaced.png"), + (images.png(1, 1, gray=True), "image/png", "1x1.png")): + att = upload(alice, chat["id"], data, ctype, name).json + check(att["status"] == "ready" and att["error_code"] is None, "not ready") + check([v["name"] for v in att["variants"]] == ["thumbnail", "crop", "optimized"], "lab5 variants missing") + check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ") + check(att["content_type"] == ctype and att["filename"] == name, "metadata differs") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta == att, "GET metadata differs from POST response") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "downloaded bytes differ") + check(got.headers["content-type"] == ctype, "wrong Content-Type on download") + check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length") + check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing") + print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes") + print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte") + + +def test_size_limit_exact_boundary(): + alice, _, _, chat = setup() + exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES) + check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB") + att = upload(alice, chat["id"], exact).json + check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed") + print(f"exactly {len(exact)} bytes (10 MiB) -> 202, then ready") + over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1) + reply = upload(alice, chat["id"], over, expected=413) + check_error(reply, "payload_too_large") + print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body") + print("PASS: the 10 MiB limit is exact") + + +def chunked_upload(user, chat_id, total, piece=64 * 1024): + """Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении.""" + conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60) + + def body(): + sent = 0 + head = images.sample_png()[:64] + yield head + sent += len(head) + while sent < total: + part = bytes(min(piece, total - sent)) + sent += len(part) + yield part + try: + conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(), + headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]}, + encode_chunked=True) + reply = conn.getresponse() + return reply.status, reply.read() + finally: + conn.close() + + +def test_size_limit_streaming(): + alice, _, _, chat = setup() + status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024) + check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}") + print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed") + call("GET", "/health/ready", 200) + print("PASS: oversized streamed body is cut off, the API keeps working") + + +def test_pixel_limit(): + alice, _, _, chat = setup() + at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей + att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json + print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 202, ready {att['id'][:8]}") + over = images.png(8001, 5000, gray=True) + check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large") + print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413") + bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01") + check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large") + print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded") + print("PASS: pixel limit is checked before decompression") + + +def test_damaged_images(): + alice, _, _, chat = setup() + png, jpeg = images.sample_png(), images.sample_jpeg() + cases = [ + ("PNG cut in half", png[:len(png) // 2], "image/png"), + ("PNG without IEND", png[:-12], "image/png"), + ("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"), + ("PNG with garbage after IEND", png + b"garbage", "image/png"), + ("PNG signature only", png[:8], "image/png"), + ("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"), + ("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"), + ("JPEG header only", jpeg[:20], "image/jpeg"), + ("random bytes", uuid.uuid4().bytes * 100, "image/png"), + ("plain text", "это не картинка".encode(), "image/jpeg"), + ] + for name, data, ctype in cases: + reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415) + check_error(reply, "unsupported_media_type") + print(f"{name} ({len(data)} bytes, {ctype}) -> 415") + check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request") + print("empty body -> 400") + print("PASS: damaged and foreign files are rejected with 415, never 500") + + +def test_png_chunk_semantics(): + """CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201. + + Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready, + хотя Pillow его отклоняет (invalid palette size). + """ + alice, bob, _, chat = setup() + from images import assemble, ihdr, zero_idat + pal = lambda entries: (b"PLTE", bytes(3 * entries)) + idat_p = zero_idat(1, 1, 8, 1) + bad = [ + ("PLTE 771 bytes = 257 entries (review case)", + assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))), + ("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)), + ("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)), + ("palette larger than bit depth allows (depth 1, 3 entries)", + assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))), + ("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)), + ("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))), + ("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))), + ("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)), + ("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)), + ("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)), + ("tRNS of wrong size for RGB (4 bytes instead of 6)", + assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))), + ("tRNS in an image with alpha channel", + assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))), + ("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))), + ("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))), + ("IDAT chunks not consecutive", + assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"), + (b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))), + ("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))), + ("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))), + ("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)), + zero_idat(1, 1, 8, 3))), + ("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))), + ("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))), + ("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))), + ("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))), + ("zTXt with broken zlib after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))), + ("zTXt text bomb: 50 MB of text in ~50 KB after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))), + ] + for name, data in bad: + check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type") + print(f"{name} ({len(data)} bytes, valid CRCs) -> 415") + good = [ + ("palette 256 entries + tRNS + tEXt and tIME after IDAT", + assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1), + (b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))), + ("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))), + ("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)), + (b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))), + ("compressed zTXt (100 KB of text) and iTXt after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)), + (b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))), + ] + for name, data in good: + att = upload(alice, chat["id"], data, filename="good.png").json + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "valid PNG changed") + print(f"{name} -> 201, downloaded byte for byte") + print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted") + + +def test_mime_substitution(): + alice, _, _, chat = setup() + cases = [ + ("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"), + ("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"), + ("GIF bytes sent as image/png", images.GIF, "image/png"), + ("PNG sent as image/gif", images.sample_png(), "image/gif"), + ("PNG sent as text/html", images.sample_png(), "text/html"), + ("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"), + ("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"), + ] + for name, data, ctype in cases: + check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type") + print(f"{name} -> 415") + att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json + check(att["content_type"] == "image/png", "content type must be normalised") + print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png") + print("PASS: the type is taken from the content; a declared type that does not match is refused") + + +def test_filenames(): + alice, _, _, chat = setup() + bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png", + "bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256] + for name in bad: + check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request") + print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400") + check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request") + print("no filename -> 400") + for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"): + att = upload(alice, chat["id"], images.sample_png(), filename=name).json + check(att["filename"] == name, "display name changed") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded") + print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only") + print("PASS: filename never becomes a path; dangerous names are refused") + + +def test_access_control(): + alice, bob, eve, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + for suffix in ("", "/content"): + reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + check_error(reply, "not_found") + call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400") + check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found") + check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found") + upload(None, chat["id"], images.sample_png(), expected=401) + print("upload into someone else's / unknown chat -> 404, without token -> 401") + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=preview", 400, token=bob["token"]) + for variant in ("original", "thumbnail", "crop", "optimized"): + call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}", 200, token=bob["token"]) + print("variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)") + print("PASS: only chat members can see an attachment") + + +def test_message_attachments(): + alice, bob, eve, chat = setup() + first = upload(alice, chat["id"], images.sample_png()).json + second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json + msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json + check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed") + print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member") + plain = post_message(bob, chat["id"], "без вложений").json + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json + history, _ = read_history(bob, chat["id"]) + check(history == [msg, plain, reused], "history differs from the confirmed messages") + print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed") + + other_chat = new_chat(alice, eve) + foreign = upload(alice, other_chat["id"], images.sample_png()).json + for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"), + ([first["id"], foreign["id"]], "own + foreign attachment"), + ([str(uuid.uuid4())], "unknown attachment_id")): + check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found") + print(f"{what} -> 404") + check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found") + print("outsider referencing an attachment of the chat -> 404") + for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"), + ([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"), + (["not-a-uuid"], "x", "bad UUID"), + ([], "", "empty text without attachments"), + ([], " ", "whitespace text without attachments")): + check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request") + print(f"{what} -> 400") + history_after, _ = read_history(bob, chat["id"]) + check(history_after == history, "a rejected message was stored") + print("PASS: only attachments of the same chat can be linked; rejected requests change nothing") + + +def test_idempotency_with_attachments(): + alice, bob, _, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json + key = new_key() + first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + check(again == first, "replay returned a different message") + check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict") + check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict") + key2 = new_key() + plain = post_message(alice, chat["id"], "текст", None, key=key2).json + check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field") + history, _ = read_history(bob, chat["id"]) + check(len(history) == 2, "replays created extra messages") + print("same key + same attachments -> same 201; other attachments -> 409; missing field == []") + print("PASS: Idempotency-Key covers attachment_ids") + + +def test_aborted_upload(): + alice, _, _, chat = setup() + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000)) + sock.close() # клиент пропал на середине тела + print("sent 1.1 MB of a declared 5 MB body and closed the connection") + att = upload(alice, chat["id"], images.sample_png()).json + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + print("the API keeps serving uploads; nothing of the aborted request is visible " + "(DB/S3 side is checked by test_consistency.py)") + print("PASS: an interrupted upload is dropped") + + +if __name__ == "__main__": + run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, + test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control, + test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..01f2251 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,103 @@ +from collections import Counter + +import images +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_attachment_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + sent = upload(alice, chat["id"], data) + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"]) + check(reply.data == data, "another replica returned different bytes") + readers.add(reply.instance) + print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}") + check(readers - {sent.instance}, "no download went to another replica") + print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_attachment_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 0000000..ff3d4ab --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,276 @@ +"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка). + +Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`), +выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает +janitor вручную. Данные синтетические, volumes не удаляются. + + python student-tests/test_consistency.py [s3|crash|orphan|aborted|all] +""" +import json +import socket +import sys +import time +import urllib.parse + +import images +from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat, + new_user, post_message, run_tests, sql_value, upload, wait_ready) + +URL = urllib.parse.urlsplit(BASE_URL) + +# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но enqueue_upload +# (lab5: UPDATE ... state='queued' + задание + outbox, COMMIT) падает — как если бы реплика умерла +# сразу после PUT в staging, до фиксации задания. +CRASH_AFTER_PUT = r''' +import json, struct, sys, tempfile, zlib +from uuid import UUID +import attachments, db, storage, uploads +chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2]) +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b"")) +f = tempfile.SpooledTemporaryFile(); f.write(data) +import hashlib +upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest()) +image = uploads.inspect(upload, "image/png") +created = {} +real_intent = db.create_upload_intent +def remember(attachment_id, *args): + created["id"], created["key"] = str(attachment_id), args[-1] + return real_intent(attachment_id, *args) +def crash(attachment_id, *args): + raise RuntimeError("simulated crash between S3 PUT and DB COMMIT") +db.create_upload_intent = remember +db.enqueue_upload = crash +db.pool.open() +try: + attachments.store(chat_id, owner_id, "crash.png", upload, image) + outcome = "stored" +except RuntimeError as exc: + outcome = str(exc) +print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"], + "object_exists": storage.exists(created["key"])})) +''' + +OBJECT_EXISTS = r''' +import json, sys, storage +print(json.dumps({"exists": storage.exists(sys.argv[1])})) +''' + +PUT_ORPHAN = r''' +import hashlib, io, json, uuid, storage +data = b"orphan object without a row in PostgreSQL" +key = storage.new_key(uuid.uuid4().hex) +storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest()) +print(json.dumps({"key": key})) +''' + +DELETE_OBJECT = r''' +import json, sys, storage +storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]})) +''' + +COUNT_OBJECTS = r''' +import json, storage +print(json.dumps({"objects": sum(1 for prefix in storage.ALL_PREFIXES for _ in storage.list_objects(prefix))})) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def object_exists(key): + return last_json(app_python(OBJECT_EXISTS, key))["exists"] + + +def backdate(where): + sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}") + + +def uploading_rows(chat_id): + return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'")) + + +def wait_download(user, att, seconds=60): + deadline = time.monotonic() + seconds + while True: + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"]) + if reply.status == 200: + return reply + check(time.monotonic() < deadline, "download did not recover after S3 start") + time.sleep(1) + + +def s3_outage(stop_action, start_action, label): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + att = upload(alice, chat["id"], data).json + before = uploading_rows(chat["id"]) + print(f"--- S3 {label}: docker compose {stop_action} s3") + compose(stop_action, "s3") + try: + started = time.monotonic() + reply = upload(alice, chat["id"], data, expected=503) + ms = (time.monotonic() - started) * 1000 + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + body = reply.data.decode().lower() + check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")), + "error leaks storage details") + print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body") + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + print("download of an existing file -> 503") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta["sha256"] == att["sha256"], "metadata changed") + post_message(alice, chat["id"], "текст без файла") + post_message(alice, chat["id"], "", [att["id"]]) + call("GET", "/health/ready", 200) + print("metadata, messages (also with an already stored attachment) and /health/ready keep working: " + "S3 is needed only for file bytes") + after = uploading_rows(chat["id"]) + print(f"uploading rows of this chat: before {before}, after the failed upload {after} " + "(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API") + finally: + print(f"docker compose {start_action} s3") + compose(start_action, "s3") + got = wait_download(bob, att) + check(got.data == data, "file changed after S3 recovery") + print("after S3 recovery the old file downloads byte for byte") + upload(alice, chat["id"], data) + print("new upload -> 201") + leftover = uploading_rows(chat["id"]) + if leftover: + backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'") + report = janitor() + print(f"janitor after grace period: {json.dumps(report)}") + check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain") + print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up") + + +def test_s3_stopped(): + s3_outage("stop", "start", "stopped (connection refused / name not resolved)") + + +def test_s3_frozen(): + s3_outage("pause", "unpause", "frozen (no answer, timeouts)") + + +def test_crash_between_s3_and_db(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"])) + check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}") + att_id, key = result["attachment_id"], result["object_key"] + state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'") + check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3") + print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3") + call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"]) + post_message(alice, chat["id"], "x", [att_id], expected=404) + print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404") + + report = janitor() + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key), + "fresh upload was deleted inside the grace period") + print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> " + "fresh row and object are kept (the upload might still be running)") + backdate(f"id = '{att_id}'") + report = janitor("--dry-run") + check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row") + print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted") + report = janitor() + check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists") + check(not object_exists(key), "object still exists") + print(f"janitor: {json.dumps(report)} -> row and object deleted") + report = janitor() + print(f"janitor again: {json.dumps(report)} -> idempotent") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment damaged by cleanup") + print("the ready attachment of the same chat is untouched") + print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period") + + +def test_orphan_objects(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + key = last_json(app_python(PUT_ORPHAN))["key"] + print(f"object {key} put into the bucket without a row in PostgreSQL") + # Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с. + short = {"UPLOAD_TIMEOUT_SECONDS": "1"} + report = janitor("--grace-seconds", "3", env=short) + check(object_exists(key), "fresh orphan deleted inside grace period") + print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept") + time.sleep(4) + report = janitor("--grace-seconds", "3", "--dry-run", env=short) + check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key), + "dry run must report and keep the orphan") + print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept") + report = janitor("--grace-seconds", "3", env=short) + check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted") + print(f"janitor: {json.dumps(report)} -> orphan deleted") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment was deleted with orphans") + print("ready attachments older than the grace period are untouched (they have rows)") + + victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json + victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'") + app_python(DELETE_OBJECT, victim_key) + report = janitor("--grace-seconds", "3", env=short) + check(report["missing_objects"] >= 1, "missing object not reported") + check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted") + reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"]) + print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects=" + f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)") + print("PASS: orphans are found and deleted only after the grace period; missing objects are reported") + + +def api_tmp_files(): + counts = {} + for container in compose_output("ps", "-q", "api").split(): + counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0) + return counts + + +def test_aborted_upload_leaves_nothing(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + objects = last_json(app_python(COUNT_OBJECTS))["objects"] + rows = int(sql_value("SELECT count(*) FROM attachments")) + for sent in (100_000, 3_000_000): + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n" + f"Content-Length: 9000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(sent)) + sock.close() + print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection") + time.sleep(2) + check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row") + check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object") + print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}") + tmp = api_tmp_files() + check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}") + print(f"temporary upload files in api containers: {tmp} (removed when the request ends)") + for container in compose_output("ps", "-q", "api").split(): + mounts = docker("inspect", "-f", "{{json .Mounts}}", container) + # файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных + storage = [m for m in json.loads(mounts) + if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)] + check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}") + print("api containers have no volumes: the original is never stored locally") + print("PASS: an interrupted upload leaves no row, no object and no local file") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db], + "orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]} + groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"] + check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]") + wait_ready() + run_tests(groups[target]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 0000000..3d71fcc --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..e6017ce --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 0000000..8eaef38 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..407dea3 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_janitor_replay.py b/student-tests/test_janitor_replay.py new file mode 100644 index 0000000..4706a93 --- /dev/null +++ b/student-tests/test_janitor_replay.py @@ -0,0 +1,154 @@ +"""Регрессия: janitor не должен удалять объекты, которые replay записывает заново. + +Дефект (найден на ревью): у failed-вложения остаются объекты под attachments//v1/ (например, worker упал +после PUT, затем попытки исчерпаны). Janitor считал их сиротами, хотя failed ещё можно повторить (исходник +в staging хранится STAGING_RETENTION_SECONDS). Повторная проверка и DELETE не были синхронизированы с replay: +janitor, приостановленный после проверки, удалял crop.jpg, только что записанный успешным replay, +а метаданные оставались ready -> /content?variant=crop -> 503. + +Сценарий воспроизводит ровно это чередование: janitor запускается в контейнере janitor с перехватом +storage.delete — перед удалением любого объекта этого вложения он печатает PAUSED и ждёт; в это время тест +делает штатный ops.py replay и ждёт ready, потом отпускает janitor. Ожидание после исправления: janitor +вообще не пытается удалить объекты failed-вложения, пока replay возможен; после replay все 4 файла — 200. +Вторая часть: когда исходник удалён по retention (replay больше невозможен — состояние конечное), +оставшиеся объекты failed-вложения удаляются, а replay отказывает. + + python student-tests/test_janitor_replay.py +Длительность ~4 мин (объекты должны стать старше grace = 120 с, минимально допустимого). +""" +import json +import subprocess +import threading +import time + +from common import ROOT, call, check, run_tests, wait_ready +from test_jobs import (clear_faults, content, fault, get, image, log, ops, probe, upload, user_and_chat, + wait_status, job) + +GRACE = 120 # минимально допустимый grace: 2 x UPLOAD_TIMEOUT_SECONDS + +HOOK = r''' +import json, pathlib, sys, time +import janitor, storage +attachment_id, grace = sys.argv[1], int(sys.argv[2]) +prefix = f"attachments/{attachment_id}/" +release = pathlib.Path("/tmp/janitor-release-" + attachment_id) +real_delete = storage.delete +deleted = [] +def delete(key, *args, **kwargs): + if key.startswith(prefix): + print("PAUSED " + key, flush=True) # проверка уже пройдена, DELETE ещё не выполнен + deadline = time.time() + 120 + while not release.exists() and time.time() < deadline: + time.sleep(0.2) + deleted.append(key) + return real_delete(key, *args, **kwargs) +storage.delete = delete +report = None +for _ in range(60): # фоновый janitor может держать advisory lock + report = janitor.run_once(grace, False) + if report is not None: + break + time.sleep(2) +print(json.dumps({"report": report, "deleted_ours": deleted}), flush=True) +''' + + +def exec_python(service, code, *args, env=None): + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode(), capture_output=True, check=False) + check(result.returncode == 0, f"{service}: {result.stderr.decode('utf-8', 'replace')[-500:]}") + return result.stdout.decode("utf-8", "replace") + + +def failed_with_leftovers(name): + """failed/retry_exhausted, у которого под attachments//v1/ остались объекты прерванной попытки.""" + clear_faults() + fault("worker_crash_after_put") # попытка 1: объекты записаны, worker убит до COMMIT + att = upload(image("thumb320"), filename=f"{name}.png") + deadline = time.monotonic() + 30 + while len(probe("inspect", att["id"])["final_keys"]) < 4: + check(time.monotonic() < deadline, "attempt 1 did not write its objects") + time.sleep(0.3) + written = time.monotonic() + fault("worker_s3_error", 4) # попытки 2-5: временная ошибка S3 -> retry_exhausted + failed = wait_status(att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", f"{name}: {failed}") + keys = probe("inspect", att["id"])["final_keys"] + check(len(keys) == 4, f"{name}: leftovers expected, got {keys}") + log(f"{name}: failed/retry_exhausted after a crash post-PUT; leftover objects {keys}") + return att, written + + +def test_janitor_does_not_delete_replayed_objects(): + user_and_chat() + a, written_a = failed_with_leftovers("race-replayable") + b, written_b = failed_with_leftovers("race-terminal") + wait = max(written_a, written_b) + GRACE + 5 - time.monotonic() # объекты ОБОИХ старше grace + if wait > 0: + log(f"waiting {wait:.0f} s until the leftovers are older than grace ({GRACE} s)") + time.sleep(wait) + + # 1. janitor проходит bucket; если решит удалить объект A — остановится перед DELETE + proc = subprocess.Popen(["docker", "compose", "exec", "-T", "janitor", "python", "-", a["id"], str(GRACE)], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + proc.stdin.write(HOOK.encode()) + proc.stdin.close() + lines = [] + reader = threading.Thread(target=lambda: lines.extend(l.decode().strip() for l in proc.stdout), daemon=True) + reader.start() + deadline = time.monotonic() + 60 + while not any(l.startswith("PAUSED") for l in lines) and proc.poll() is None: + check(time.monotonic() < deadline, "janitor pass did not finish") + time.sleep(0.2) + paused = [l for l in lines if l.startswith("PAUSED")] + log(f"janitor pass over the bucket: {'paused before DELETE of ' + paused[0][7:] if paused else 'did not try to delete any object of the replayable failed attachment'}") + + # 2. штатный replay, пока janitor стоит (или уже прошёл) + j = job(a["id"]) + check("replayed" in ops("replay", j["job_id"], "--reason", "janitor race regression"), "replay refused") + wait_status(a["id"], "ready", 60) + log(f"ops replay -> job {j['job_id']} ready") + + # 3. отпускаем janitor и ждём конца прохода + exec_python("janitor", "import pathlib, sys\npathlib.Path('/tmp/janitor-release-' + sys.argv[1]).touch()\n", a["id"]) + proc.wait(timeout=120) + reader.join(timeout=5) + result = json.loads([l for l in lines if l.startswith("{")][-1]) + check(result["report"] is not None, "janitor pass did not run") + + # 4. результат replay цел + for variant in ("original", "thumbnail", "crop", "optimized"): + content(a["id"], variant, expected=200) + info = probe("inspect", a["id"]) + check(info["final_keys"] == ["v1/crop.jpg", "v1/optimized.webp", "v1/original", "v1/thumbnail.jpg"], + f"objects after the janitor pass: {info['final_keys']}") + check(not result["deleted_ours"], f"janitor deleted objects of the replayed attachment: {result['deleted_ours']}") + log("after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; " + f"janitor report: orphans {result['report']['orphan_objects']}, deleted {result['report']['orphan_objects_deleted']}") + + # 5. B всё ещё можно повторить -> его остатки тоже не тронуты + check(len(probe("inspect", b["id"])["final_keys"]) == 4, "leftovers of a replayable failed attachment deleted") + + # 6. retention истёк -> исходника нет, replay невозможен (конечное состояние) -> остатки удаляются + terminal = exec_python("janitor", "import janitor, json, sys, time\n" + "for _ in range(60):\n" + " r = janitor.run_once(int(sys.argv[1]), False)\n" + " if r is not None:\n break\n time.sleep(2)\n" + "print(json.dumps(r))\n", GRACE, env={"STAGING_RETENTION_SECONDS": "1"}) + left = probe("inspect", b["id"])["final_keys"] + check(left == [], f"leftovers of a terminal failed attachment must be swept, got {left}; janitor: " + f"{terminal.strip().splitlines()[-1]}") + refused = ops("replay", job(b["id"])["job_id"]) + check("refused" in refused and "staging" in refused["refused"], f"replay must be refused: {refused}") + check(get(b["id"])["status"] == "failed", "terminal attachment must stay failed") + log(f"retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of " + f"the failed attachment swept; replay refused: \"{refused['refused']}\"") + + +if __name__ == "__main__": + wait_ready() + run_tests([test_janitor_does_not_delete_replayed_objects]) diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 0000000..b382ee2 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,567 @@ +"""Lab5: асинхронная обработка вложений — варианты, сохранность, отказы, повторы, DLQ, replay. + +Нужен запущенный стенд из compose.yaml с FAULT_INJECTION=1 (по умолчанию на учебном стенде). +Тест управляет только своими синтетическими данными и сервисами через `docker compose stop|start|restart`, +volumes не удаляет. Сбои внедряются файлами в volume faults (faults.py), SQL — только чтение. + + python student-tests/test_jobs.py [сценарий ...] # без аргументов — все по порядку + +Сценарии: variants limits worker-down broker-down outbox-crash crash-after-put crash-after-commit + lease retry exhausted-replay permanent concurrency dlq +""" +import base64 +import json +import sys +import time +import urllib.parse +from pathlib import Path + +from common import (call, check, compose, compose_output, new_chat, new_user, post_message, psql, run_tests, + send, sql_value, wait_ready) + +HERE = Path(__file__).resolve().parent +PROBE = (HERE / "worker_probe.py").read_text(encoding="utf-8") +FINAL_KEYS = ["v1/crop.jpg", "v1/optimized.webp", "v1/original", "v1/thumbnail.jpg"] +LEASE = 15 # JOB_LEASE_SECONDS стенда +TIMEOUT = 60 # JOB_TIMEOUT_SECONDS стенда +MAX_ATTEMPTS = 5 # JOB_MAX_ATTEMPTS стенда + +STATE = {} + + +# ---------- инфраструктура ---------- + +def in_container(service, code, *args): + """python - в контейнере service, код через stdin. Последняя строка stdout — JSON. + Если service остановлен (сценарии с выключенным worker) — в одноразовом контейнере того же сервиса.""" + import subprocess + from common import ROOT + running = subprocess.run(["docker", "compose", "ps", "--status", "running", "-q", service], cwd=ROOT, + capture_output=True, check=False).stdout.strip() + launch = (["exec", "-T", service, "python"] if running + else ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service]) + result = subprocess.run(["docker", "compose", *launch, "-", *map(str, args)], cwd=ROOT, + input=code.encode("utf-8"), capture_output=True, check=False) + check(result.returncode == 0, f"{service}: {result.stderr.decode('utf-8', 'replace')[-500:]}") + return json.loads(result.stdout.decode("utf-8").strip().splitlines()[-1]) + + +def probe(*args): + return in_container("worker", PROBE, *args) + + +def ops(*args): + import subprocess + from common import ROOT + result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], + cwd=ROOT, capture_output=True, check=False) + check(result.returncode == 0, f"ops {args[0]}: {result.stderr.decode('utf-8', 'replace')[-400:]}") + return json.loads(result.stdout.decode("utf-8")) + + +def fault(point, times=1): + """Взвести точку сбоя: times файлов /faults/. (каждое срабатывание удаляет один файл).""" + code = ("import pathlib, sys, time\n" + "for n in range(int(sys.argv[2])):\n" + " pathlib.Path('/faults', f'{sys.argv[1]}.{time.time_ns()}_{n}').touch()\n" + "print('{}')\n") + in_container("worker", code, point, times) + + +def clear_faults(): + code = "import pathlib\nfor p in pathlib.Path('/faults').glob('*'):\n p.unlink()\nprint('{}')\n" + in_container("worker", code) + + +def image(kind): + return base64.b64decode(probe("make", kind)["b64"]) + + +def user_and_chat(): + if "alice" not in STATE: + STATE["alice"] = new_user("alice") + STATE["bob"] = new_user("bob") + STATE["chat"] = new_chat(STATE["alice"], STATE["bob"], title="Lab5 jobs") + return STATE["alice"], STATE["bob"], STATE["chat"] + + +def upload(data, content_type="image/png", filename="probe.png", expected=202): + alice, _bob, chat = user_and_chat() + query = "?filename=" + urllib.parse.quote(filename, safe="") + att = call("POST", f"/api/v1/chats/{chat['id']}/attachments{query}", expected, raw=data, token=alice["token"], + headers={"Content-Type": content_type}, timeout=60).json + if expected == 202: + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, + f"202 must be a queued snapshot, got {att['status']}") + return att + + +def get(att_id, user=None): + user = user or STATE["alice"] + return call("GET", f"/api/v1/attachments/{att_id}", 200, token=user["token"]).json + + +def wait_status(att_id, statuses, seconds=60): + statuses = (statuses,) if isinstance(statuses, str) else statuses + deadline = time.monotonic() + seconds + while True: + att = get(att_id) + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"{att_id[:8]}: still {att['status']} after {seconds} s") + time.sleep(0.3) + + +def content(att_id, variant="original", expected=200): + return call("GET", f"/api/v1/attachments/{att_id}/content?variant={variant}", expected, + token=STATE["bob"]["token"]) + + +def job(att_id): + rows = sql_value(f"SELECT job_id || '|' || status || '|' || attempt || '|' || coalesce(error_code, '-') " + f"FROM attachment_jobs WHERE attachment_id = '{att_id}'") + check(rows and "\n" not in rows, f"expected exactly one job for {att_id}") + job_id, status, attempt, error = rows.split("|") + return {"job_id": job_id, "status": status, "attempt": int(attempt), "error_code": error} + + +def events(job_id): + out = sql_value(f"SELECT attempt || ':' || event || ':' || coalesce(detail, '') FROM job_events " + f"WHERE job_id = '{job_id}' ORDER BY id") + return out.splitlines() if out else [] + + +def names(evs): + return [e.split(":")[1] for e in evs] + + +def assert_one_result(att_id, label=""): + """Инвариант идемпотентности: ровно один набор вариантов и ровно 4 объекта под конечным prefix.""" + info = probe("inspect", att_id) + check(info["state"] == "ready", f"{label}: not ready in DB") + check(info["variant_rows"] == 3, f"{label}: {info['variant_rows']} variant rows, expected 3") + check(info["final_keys"] == FINAL_KEYS, f"{label}: final objects {info['final_keys']}") + check(info["original_sha_ok"] and info["original_size_ok"], f"{label}: original differs from upload") + check(all(v["sha_ok"] and v["meta_size_ok"] for v in info["variants"].values()), f"{label}: variant mismatch") + return info + + +def outbox_rows(job_id): + out = sql_value(f"SELECT attempt || ':' || (published_at IS NOT NULL) FROM outbox WHERE job_id = '{job_id}' " + f"ORDER BY id") + return out.splitlines() if out else [] + + +def queue_depth(): + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_queues", "-p", "messenger", + "name", "messages", "messages_unacknowledged") + depth = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[1].isdigit(): + depth[parts[0]] = (int(parts[1]), int(parts[2])) + return depth + + +def log(text): + print(text, flush=True) + + +# ---------- сценарии ---------- + +def test_variants(): + """thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.""" + alice, bob, chat = user_and_chat() + cases = { + "thumb320": ("image/png", {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [320, 200]}), + "exif6": ("image/jpeg", {"thumbnail": [192, 256], "crop": [128, 128], "optimized": [300, 400]}), + "alpha": ("image/png", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [300, 200]}), + "tiny": ("image/png", {"thumbnail": [40, 30], "crop": [128, 128], "optimized": [40, 30]}), + "large": ("image/jpeg", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [1280, 853]}), + "strip": ("image/png", {"thumbnail": [256, 1], "crop": [128, 128], "optimized": [1280, 1]}), + } + for kind, (ctype, sizes) in cases.items(): + data = image(kind) + att = upload(data, ctype, f"{kind}.{'jpg' if ctype == 'image/jpeg' else 'png'}") + ready = wait_status(att["id"], "ready") + meta = {v["name"]: v for v in ready["variants"]} + check(set(meta) == {"thumbnail", "crop", "optimized"}, f"{kind}: variants {sorted(meta)}") + for name, size in sizes.items(): + check([meta[name]["width"], meta[name]["height"]] == size, + f"{kind}/{name}: {meta[name]['width']}x{meta[name]['height']}, expected {size}") + check(meta["thumbnail"]["content_type"] == "image/jpeg" and meta["crop"]["content_type"] == "image/jpeg" + and meta["optimized"]["content_type"] == "image/webp", f"{kind}: wrong variant content types") + original = content(att["id"]) + check(original.data == data, f"{kind}: original is not byte-identical") + for name in ("thumbnail", "crop", "optimized"): + reply = content(att["id"], name) + check(reply.headers["content-type"] == meta[name]["content_type"], f"{kind}/{name}: Content-Type") + info = assert_one_result(att["id"], kind) + for name, v in info["variants"].items(): + check(v["exif_tags"] == 0 and not v["has_xmp"], f"{kind}/{name}: metadata not stripped") + if kind == "exif6": + top = info["variants"]["thumbnail"]["pixel_top_center"] + check(top[0] > 200 and top[1] < 60, f"exif6: orientation not applied, top pixel {top}") + if kind == "alpha": + left = info["variants"]["thumbnail"]["pixel_left_center"] + right = info["variants"]["thumbnail"]["pixel_right_center"] + check(left[:3] == [255, 255, 255] and right[2] > 200, f"alpha: not composited on white {left} {right}") + check(info["variants"]["optimized"]["mode"] == "RGBA", "alpha: optimized WebP lost transparency") + log(f"{kind}: " + ", ".join(f"{n} {s[0]}x{s[1]}" for n, s in sizes.items()) + + f"; original byte-identical; EXIF/XMP in variants: 0; S3 keys {info['final_keys']}") + # Отсутствующий вариант у ready -> 404 (нет такого имени в контракте -> 400), queued -> 409 проверяется ниже. + content(att["id"], "unknown", expected=400) + log("unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload") + + +def test_limits(): + """Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.""" + alice, _bob, chat = user_and_chat() + big = b"\x89PNG\r\n\x1a\n" + b"\0" * (10 * 1024 * 1024) + call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=big.png", 413, raw=big, token=alice["token"], + headers={"Content-Type": "image/png"}, timeout=60) + log("10 MiB + 8 bytes -> 413 before any job is created") + report = probe("imaging-limits") + at, over = report["at_limit_40mpx"], report["over_limit_max_10mpx"] + check(at["exit_code"] == 0, f"40 Mpx PNG at the limit must be processed, exit {at['exit_code']}") + check(over["exit_code"] == 4, f"worker-side pixel limit must reject (exit 4), got {over['exit_code']}") + check(at["peak_rss_mib_max_children"] < report["memory_limit_mib"], + f"peak RSS {at['peak_rss_mib_max_children']} MiB exceeds RLIMIT_AS {report['memory_limit_mib']} MiB") + log(f"imaging.py on 8000x5000 PNG: exit 0, peak RSS {at['peak_rss_mib_max_children']} MiB " + f"(RLIMIT_AS {report['memory_limit_mib']} MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)") + # Память: процесс обработки упирается в RLIMIT_AS -> MemoryError -> постоянная ошибка resource_limit. + fault("imaging_oom") + att = upload(image("thumb320"), filename="oom.png") + failed = wait_status(att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "resource_limit", + f"oom: expected failed/resource_limit, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == 1, "resource_limit is permanent: no retries expected") + log(f"memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived") + STATE["oom_job"] = j["job_id"] + # Время: зависший процесс обработки убивается по JOB_TIMEOUT_SECONDS -> временная ошибка -> retry -> ready. + fault("imaging_hang") + att = upload(image("thumb320"), filename="hang.png") + started = time.monotonic() + ready = wait_status(att["id"], "ready", TIMEOUT + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check("lease_expired" not in names(evs), f"a long-running (alive) attempt must keep its lease: {evs}") + check(any("processing_timeout" in e for e in evs) and j["attempt"] == 2, f"timeout not handled as retry: {evs}") + log(f"imaging hang -> killed after {TIMEOUT} s (processing_timeout, temporary), lease renewed meanwhile " + f"(no lease_expired), attempt 2 ready after {time.monotonic() - started:.0f} s") + + +def test_worker_down(): + """Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.""" + alice, bob, chat = user_and_chat() + compose("stop", "worker") + log("worker stopped") + data = image("thumb320") + att = upload(data, filename="offline.png") + log(f"upload while worker is down -> 202 queued, attachment {att['id']}") + j = job(att["id"]) + source_key = sql_value(f"SELECT source_key FROM attachments WHERE id = '{att['id']}'") + check(source_key.startswith("staging/"), "source must be in the durable staging prefix") + check(probe("exists", source_key)["exists"], "staging object must exist before 202") + msg = post_message(alice, chat["id"], "photo while worker is down", [att["id"]]).json + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + content(att["id"], expected=409) + content(att["id"], "thumbnail", expected=409) + send(bob, chat["id"], "other messages keep working") + deadline = time.monotonic() + 10 + while outbox_rows(j["job_id"]) != ["1:true"]: + check(time.monotonic() < deadline, "dispatcher did not publish the job") + time.sleep(0.2) + deadline = time.monotonic() + 15 # счётчики quorum-очереди в rabbitmqctl обновляются раз в несколько секунд + while (queue_depth().get("attachments.process.v1") or (0, 0))[0] < 1: + check(time.monotonic() < deadline, "published message is not visible in the queue") + time.sleep(0.5) + depth = queue_depth().get("attachments.process.v1") + log(f"message with attachment 201, content 409, other messages 201; job {j['job_id']} published, " + f"queue messages/unacked = {depth}") + compose("restart", "rabbitmq") + compose("restart", "api") + wait_ready() + depth_after = queue_depth().get("attachments.process.v1") + check(depth_after and depth_after[0] >= 1, f"message lost after broker restart: {depth_after}") + check(get(att["id"])["status"] == "queued", "job must still be queued after restarts") + log(f"rabbitmq and api restarted with volumes; queue still holds {depth_after[0]} message(s); status queued") + compose("start", "worker") + ready = wait_status(att["id"], "ready", 60) + j2 = job(att["id"]) + check(j2["job_id"] == j["job_id"] and j2["status"] == "ready", "same job_id must finish") + thumb = content(att["id"], "thumbnail") + check(len(thumb.data) == {v["name"]: v for v in ready["variants"]}["thumbnail"]["size_bytes"], "thumbnail size") + info = assert_one_result(att["id"], "worker-down") + check(not info["staging_present"] and not probe("exists", source_key)["exists"], + "staging must be cleaned after the safe commit") + log(f"worker started -> same job {j['job_id']} ready, thumbnail downloadable; staging object deleted") + + +def test_broker_down(): + """Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.""" + compose("stop", "rabbitmq") + log("rabbitmq stopped") + att = upload(image("thumb320"), filename="no-broker.png") + call("GET", "/health/ready", 200) + j = job(att["id"]) + time.sleep(3) + check(outbox_rows(j["job_id"]) == ["1:false"], "outbox row must wait while the broker is down") + log(f"upload -> 202 and /health/ready 200 without broker; outbox row pending for job {j['job_id']}") + compose("start", "rabbitmq") + wait_status(att["id"], "ready", 90) + check(outbox_rows(j["job_id"]) == ["1:true"], "outbox row must be published after recovery") + assert_one_result(att["id"], "broker-down") + log("rabbitmq started -> dispatcher published the pending row -> ready") + + +def test_outbox_crash(): + """Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.""" + fault("dispatcher_crash_before_publish") + att = upload(image("thumb320"), filename="crash-before-publish.png") + ready = wait_status(att["id"], "ready", 60) + j = job(att["id"]) + logs = compose_output("logs", "--tail", "400", "dispatcher") + check("FAULT INJECTION: dispatcher_crash_before_publish" in logs, "fault did not fire") + check(outbox_rows(j["job_id"]) == ["1:true"] and j["attempt"] == 1, "outbox must publish exactly the same job") + assert_one_result(att["id"], "crash-before-publish") + log(f"dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job " + f"{j['job_id']} ready, attempt 1, no manual action") + fault("dispatcher_crash_after_confirm") + att = upload(image("thumb320"), filename="crash-after-confirm.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + deadline = time.monotonic() + 20 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"duplicate delivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + assert_one_result(att["id"], "crash-after-confirm") + log(f"dispatcher killed after publisher confirm, before marking outbox -> message published twice; " + f"worker: {names(events(j['job_id']))} -> one result set") + + +def test_crash_after_put(): + """Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.""" + fault("worker_crash_after_put") + att = upload(image("thumb320"), filename="crash-after-put.png") + wait_status(att["id"], "ready", LEASE + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check(names(evs).count("claimed") == 2 and "lease_expired" in names(evs), f"unexpected history {evs}") + assert_one_result(att["id"], "crash-after-put") + log(f"worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired " + f"after {LEASE} s, attempt {j['attempt']} rewrote the same keys -> one set; events {names(evs)}") + + +def test_crash_after_commit(): + """Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.""" + fault("worker_crash_after_commit") + att = upload(image("thumb320"), filename="crash-after-commit.png") + wait_status(att["id"], "ready", 30) + j = job(att["id"]) + deadline = time.monotonic() + 30 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"redelivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + evs = events(j["job_id"]) + check(names(evs).count("ready") == 1 and j["attempt"] == 1, f"ready must be recorded once: {evs}") + assert_one_result(att["id"], "crash-after-commit") + deadline = time.monotonic() + 20 # счётчики quorum-очереди обновляются с задержкой + while queue_depth().get("attachments.process.v1") != (0, 0): + check(time.monotonic() < deadline, f"message left in the queue: {queue_depth()}") + time.sleep(0.5) + log(f"worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op ({evs[-1]}); " + f"queue empty (0 ready, 0 unacked)") + STATE["staging_check"] = att["id"] + + +def test_lease(): + """Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; + результат зависшего отклоняется fencing-токеном, ready не откатывается.""" + fault("worker_freeze") + att = upload(image("thumb320"), filename="frozen.png") + started = time.monotonic() + wait_status(att["id"], "ready", LEASE + 40) + took = time.monotonic() - started + j = job(att["id"]) + deadline = time.monotonic() + 3 * LEASE + 20 + while "stale_result_rejected" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"stale result of the frozen attempt not seen: {events(j['job_id'])}") + time.sleep(0.5) + evs = events(j["job_id"]) + check(get(att["id"])["status"] == "ready", "late result must not roll ready back") + check(names(evs).count("ready") == 1, f"exactly one ready: {evs}") + assert_one_result(att["id"], "lease") + log(f"frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after " + f"{took:.0f} s; frozen attempt woke up -> stale_result_rejected; events {names(evs)}") + + +def test_retry(): + """Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.""" + fault("worker_s3_error", 2) + att = upload(image("thumb320"), filename="retry.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + evs = [e for e in events(j["job_id"]) if ":retry_scheduled:" in e] + check(len(evs) == 2 and j["attempt"] == 3, f"expected 2 retries, got {evs}") + delays = [float(e.split("delay=")[1].rstrip("s")) for e in evs] + check(1.0 <= delays[0] <= 2.0 and 2.0 <= delays[1] <= 4.0, f"backoff out of range: {delays}") + assert_one_result(att["id"], "retry") + log(f"2 x storage_unavailable -> retries after {delays[0]:.2f} s and {delays[1]:.2f} s (equal jitter: " + f"1..2 s, 2..4 s), attempt 3 ready") + + +def test_exhausted_replay(): + """Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.""" + alice, _bob, chat = user_and_chat() + fault("worker_s3_error", MAX_ATTEMPTS) + att = upload(image("thumb320"), filename="exhausted.png") + failed = wait_status(att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", + f"expected failed/retry_exhausted, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == MAX_ATTEMPTS, f"attempts {j['attempt']}") + content(att["id"], expected=409) + post_message(alice, chat["id"], "", [att["id"]], expected=409) + registry = {row["job_id"]: row for row in ops("failed")} + check(registry.get(j["job_id"], {}).get("replayable") is True, "failed job must be in the registry") + log(f"{MAX_ATTEMPTS} temporary errors -> failed/retry_exhausted (public error_code), content 409, " + f"attach 409, listed by ops.py failed (replayable)") + replay = ops("replay", j["job_id"], "--reason", "s3 recovered") + check("replayed" in replay, f"replay refused: {replay}") + wait_status(att["id"], "ready", 60) + assert_one_result(att["id"], "replay") + j2 = job(att["id"]) + refused = ops("replay", j["job_id"]) + check("refused" in refused, "replay of a ready job must be refused") + again = job(att["id"]) + check(again == j2, "refused replay must not change the job") + check(names(events(j["job_id"])).count("replayed") == 1, "audit must contain the replay once") + log(f"ops replay -> same job_id {j['job_id']} attempt {j2['attempt']} ready; second replay refused: " + f"\"{refused['refused']}\"") + + +def test_permanent(): + """Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.""" + fault("imaging_invalid") + att = upload(image("thumb320"), filename="invalid.png") + failed = wait_status(att["id"], ("failed", "ready"), 30) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", f"got {failed}") + j = job(att["id"]) + check(j["attempt"] == 1 and "retry_scheduled" not in names(events(j["job_id"])), "no retries for permanent") + log("invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)") + + +def test_concurrency(): + """Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.""" + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_consumers", "-p", "messenger", + "queue_name", "prefetch_count", "ack_required") + lines = [line.split() for line in out.splitlines() if line.strip()] + header = lines[0] if lines and lines[0][0] == "queue_name" else ["queue_name", "prefetch_count", "ack_required"] + consumers = [dict(zip(header, line)) for line in lines if line[0] == "attachments.process.v1"] + check(len(consumers) == 2 and all(c["prefetch_count"] == "1" and c["ack_required"] == "true" + for c in consumers), f"consumers {out}") + # Копим очередь при остановленном worker, затем запускаем: работы больше, чем потоков. + compose("stop", "worker") + data = image("large") + ids = [upload(data, "image/jpeg", f"burst{n}.jpg")["id"] for n in range(8)] + id_list = ", ".join(f"'{i}'" for i in ids) + compose("start", "worker") + started = time.monotonic() + peak, samples = 0, 0 + while True: + busy, ready = map(int, sql_value( + f"SELECT count(*) FILTER (WHERE status = 'processing') || '|' || count(*) FILTER (WHERE status = 'ready') " + f"FROM attachment_jobs WHERE attachment_id IN ({id_list})").split("|")) + peak, samples = max(peak, busy), samples + 1 + if ready == len(ids): + break + check(time.monotonic() - started < 120, "burst not processed in 120 s") + time.sleep(0.05) + took = time.monotonic() - started + check(peak == 2, f"peak {peak} jobs in processing, expected exactly WORKER_CONCURRENCY=2") + log(f"2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in {took:.1f} s after " + f"worker start, peak processing {peak} of 8 ({samples} samples)") + + +def test_dlq(): + """Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.""" + publish = r''' +import json, sys, uuid +import broker, pipeline +p = broker.Publisher() +p.open().basic_publish(pipeline.EXCHANGE, pipeline.ROUTING_KEY, b'{"not":"a job"}', mandatory=True) +p.publish(pipeline.build_message(uuid.uuid4(), uuid.uuid4(), 1, 1, None)) +p.close() +print("{}") +''' + before = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + in_container("dispatcher", publish) + deadline = time.monotonic() + 15 + while queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] < before + 2: + check(time.monotonic() < deadline, "messages did not reach the DLQ") + time.sleep(0.3) + peek = ops("dlq", "--limit", "50") + reasons = {(m["invalid"] is not None, m["dead_letter_reason"]) for m in peek} + check((True, "rejected") in reasons, f"invalid message not in DLQ: {peek}") + replayed = ops("dlq-replay", "--limit", "50") + kept = [m for m in replayed if m["action"] == "keep"] + check(len(kept) >= 2, "invalid/unknown messages must be kept, not executed") + after = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + check(after >= before + 2, "dlq-replay must not drop messages it cannot handle") + log(f"schema-invalid message and unknown job_id -> reject -> DLQ ({after} message(s)); " + f"dlq-replay kept them: {[m['note'] for m in kept][:2]}") + + +def test_staging_cleanup(): + """Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).""" + att_id = STATE.get("staging_check") + if not att_id: + log("skip: run crash-after-commit first") + return + key = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + if key: + log(f"staging source still present after the crash (worker died before cleanup); waiting 60 s window") + time.sleep(62) + from common import janitor + report = janitor() + # фоновый janitor (проход раз в JANITOR_INTERVAL_SECONDS) мог успеть первым в окне 62 с — проверяем результат + left = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + check(not left, f"staging source still referenced: {report}") + check(not probe("exists", key)["exists"], "staging object must be gone") + by = "this pass" if report["staging_ready_deleted"] else "the background janitor pass" + log(f"janitor removed the staging source of the ready attachment ({by}); source_key NULL, object gone") + else: + log("staging already cleaned by the worker") + + +SCENARIOS = { + "variants": test_variants, "limits": test_limits, "worker-down": test_worker_down, + "broker-down": test_broker_down, "outbox-crash": test_outbox_crash, "crash-after-put": test_crash_after_put, + "crash-after-commit": test_crash_after_commit, "staging": test_staging_cleanup, "lease": test_lease, + "retry": test_retry, "exhausted-replay": test_exhausted_replay, "permanent": test_permanent, + "concurrency": test_concurrency, "dlq": test_dlq, +} + + +def main(): + chosen = sys.argv[1:] or list(SCENARIOS) + wait_ready() + clear_faults() + tests = [] + for name in chosen: + def run(name=name): + log(f"--- {name}: {SCENARIOS[name].__doc__.splitlines()[0]}") + try: + SCENARIOS[name]() + finally: + compose("start", "worker") # идемпотентно: стенд остаётся рабочим даже после FAIL + compose("start", "rabbitmq") + run.__name__ = name + tests.append(run) + run_tests(tests) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 0000000..4af44fa --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) diff --git a/student-tests/test_s3_security.py b/student-tests/test_s3_security.py new file mode 100644 index 0000000..ece71a3 --- /dev/null +++ b/student-tests/test_s3_security.py @@ -0,0 +1,146 @@ +"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен. + +Нужен запущенный стенд из compose.yaml. +""" +import json +import socket + +import images +from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload + +# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API. +ANONYMOUS = r''' +import json, sys, urllib.request, urllib.error +import settings +base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET +def status(method, url, data=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r: + return r.status, r.read(300).decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read(300).decode("utf-8", "replace") +out = {} +for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None), + ("head_object", "HEAD", base + "/" + sys.argv[1], None), + ("list_bucket", "GET", base + "?list-type=2", None), + ("put_object", "PUT", base + "/originals/anonymous-test", b"x"), + ("get_bucket_policy", "GET", base + "?policy", None)): + code, body = status(method, url, data) + out[name] = {"status": code, "s3_code": body.split("")[1].split("")[0] if "" in body else ""} +print(json.dumps(out)) +''' + +# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json). +API_KEY_RIGHTS = r''' +import io, json, sys, uuid +from botocore.exceptions import ClientError +import settings, storage +c, bucket = storage._client, settings.S3_BUCKET +def attempt(name, fn): + try: + fn(); return "allowed" + except ClientError as e: + return "denied: " + e.response.get("Error", {}).get("Code", "?") +public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) +key = "originals/" + uuid.uuid4().hex +result = { + "get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()), + "put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")), + "delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)), + "list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)), + "create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])), + "make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)), + "read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)), + "delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)), + "change versioning": attempt("versioning", lambda: c.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})), +} +try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket + result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", [])) +except ClientError as e: + result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?") +print(json.dumps(result)) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + att = upload(alice, chat["id"], images.sample_png()).json + key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + return alice, bob, att, key + + +def test_bucket_is_private(): + alice, bob, att, key = prepare() + result = last_json(app_python(ANONYMOUS, key)) + for name, reply in result.items(): + print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}") + check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}") + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + print("the same file through the API for a chat member -> 200") + print("PASS: the bucket is private: no anonymous read, list, write or policy read") + + +def test_api_key_is_limited(): + alice, bob, att, key = prepare() + result = last_json(app_python(API_KEY_RIGHTS, key)) + allowed = {"get own object", "put object", "delete object", "list bucket"} + visible = result.pop("visible buckets") + print(f"API key: {'list buckets':<32} -> {visible}") + check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets") + for name, outcome in result.items(): + print(f"API key: {name:<32} -> {outcome}") + if name in allowed: + check(outcome == "allowed", f"API key must be allowed to {name}") + else: + check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}") + print("PASS: the API key can only get/put/delete/list objects of its own bucket; " + "bucket admin, policy and other buckets are denied") + + +def test_no_open_ports_or_console(): + config = json.loads(compose_output("config", "--format", "json")) + s3 = config["services"]["s3"] + check(not s3.get("ports"), "s3 must not publish ports") + check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, " + "filer UI 8888 are reachable only inside the stand)") + for port in (8333, 9333, 8888, 19333): + sock = socket.socket() + sock.settimeout(1) + refused = sock.connect_ex(("127.0.0.1", port)) != 0 + sock.close() + check(refused, f"127.0.0.1:{port} is reachable from the host") + print(f"127.0.0.1:{port} from the host -> connection refused") + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()} + for name, items in sorted(secrets.items()): + print(f"secrets of {name}: {items or '-'}") + for name in ("api", "janitor"): + check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"}, + f"{name} must not get S3 admin keys or the key config") + check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key") + print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init") + + +def test_bootstrap_is_idempotent(): + alice, bob, att, key = prepare() + output = compose_output("run", "--rm", "--no-deps", "s3-init") + print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip())) + check("bucket is private" in output, "bootstrap did not confirm a private bucket") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "file changed after repeated bootstrap") + upload(alice, att["chat_id"], images.sample_png()) + print("after the repeated bootstrap: old file downloads, new upload works") + print("PASS: bucket/user/policy bootstrap can be repeated safely") + + +if __name__ == "__main__": + run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console, + test_bootstrap_is_idempotent]) diff --git a/student-tests/test_telemetry.py b/student-tests/test_telemetry.py new file mode 100644 index 0000000..f6442fd --- /dev/null +++ b/student-tests/test_telemetry.py @@ -0,0 +1,258 @@ +"""Lab6: связь запроса, структурного лога и трассы; сквозная трасса upload -> worker, включая retry. + + python student-tests/test_telemetry.py # все сценарии + python student-tests/test_telemetry.py upload-chain # один (список SCENARIOS ниже) + +Нужен запущенный стенд (compose.yaml, FAULT_INJECTION=1 для retry/replay). Клиент сам задаёт W3C +traceparent запроса, поэтому трасса находится в Tempo по известному trace_id, без поиска «похожей». +Тест печатает дерево спанов (без данных пользователя) — это доказательство для отчёта. + +Сценарии: + request уровень 3: POST сообщения -> server-спан API с дочерними спанами PostgreSQL/Redis, + запись лога event=http.request с тем же trace_id/span_id и request_id из X-Request-Id; + текст сообщения, пароль и токен не попали ни в логи (Loki), ни в атрибуты спанов. + upload-chain уровень 4: одна трасса API(server) -> create(producer, outbox) -> dispatcher publish + (producer) -> worker process (consumer) -> S3 PutObject x4 и COMMIT ready; логи worker + с тем же trace_id. + retry-chain уровень 4: временная ошибка S3 в worker (fault worker_s3_error) -> в той же трассе + попытка 1 (ERROR, outcome=retry) -> новая публикация -> попытка 2 (ready). + replay-link уровень 4: попытки исчерпаны -> failed -> ops.py replay создаёт НОВУЮ трассу со + ссылкой (link) на исходную загрузку, worker завершает её в ready. + history уровень 3: после restart prometheus/loki/tempo (volumes сохранены) прежние трасса, + логи и метрика по-прежнему находятся. +""" +import json +import subprocess +import sys +import time +import uuid + +import obs +from common import ROOT, call, check, compose, new_chat, new_user, run_tests, wait_attachment +from images import png + +LOKI_WAIT = 60 +TRACE_WAIT = 60 + + +def upload_traced(user, chat_id, data, traceparent, filename="trace.png"): + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", 202, raw=data, + token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent}) + return reply + + +def by_name(spans, name, service=None): + return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)] + + +def parent_of(spans, span): + return next((s for s in spans if s["span_id"] == span["parent_id"]), None) + + +def fault(point, count=1): + for n in range(count): + subprocess.run(["docker", "compose", "exec", "-T", "worker", "sh", "-c", f"touch /faults/{point}.{n + 1}"], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def ops(*args): + result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], + cwd=ROOT, capture_output=True, check=True) + return json.loads(result.stdout.decode("utf-8")) + + +# ---------- сценарии ---------- + +def test_request(): + user = new_user("trace") + chat = new_chat(user, title="Trace chat") + marker = f"secret-text-{uuid.uuid4().hex}" + trace_id, traceparent = obs.new_traceparent() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": marker}, + headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}) + request_id = reply.headers["x-request-id"] + check(reply.headers.get("x-trace-id") == trace_id, "X-Trace-Id must continue the client's traceparent") + print(f"POST message: 201, X-Request-Id={request_id}, trace_id={trace_id} (client traceparent)") + + spans = obs.wait_trace(trace_id, lambda s: by_name(s, "POST /api/v1/chats/{chat_id}/messages", "messenger-api") + and any(x["kind"] == "CLIENT" and x["name"] == "INSERT" for x in s), TRACE_WAIT) + server = by_name(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")[0] + check(server["kind"] == "SERVER" and server["attrs"].get("http.response.status_code") == 201, "server span") + check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span") + db = [s for s in spans if s["parent_id"] == server["span_id"] and "db.statement" in s["attrs"] + or "db.query.text" in s["attrs"]] + check(db, "PostgreSQL client spans under the server span") + check(by_name(spans, "redis GET", "messenger-api"), "Redis session lookup span") + print(obs.span_tree(spans)) + + rows = obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}" | event="http.request"', + 1, LOKI_WAIT) + labels, line = rows[0] + check(labels.get("request_id") == request_id, "log request_id == X-Request-Id") + check(labels.get("span_id") == server["span_id"], "log span_id == server span id") + for field in ("service_name", "severity_text", "event", "trace_id", "span_id"): + check(labels.get(field), f"log field {field} is missing") + print(f"Loki: {line!r} event={labels['event']} severity={labels['severity_text']} " + f"service={labels['service_name']} trace_id={labels['trace_id']} span_id={labels['span_id']} " + f"request_id={labels['request_id']}") + + # Ни текст сообщения, ни пароль, ни токен не попали в телеметрию. + for secret, what in ((marker, "message text"), (user["password"], "password"), (user["token"][:24], "token")): + found = obs.loki_query(f'{{service_name=~"messenger-.*"}} |= `{secret}`', minutes=10, limit=5) + check(not found, f"{what} found in Loki logs") + check(not any(secret in str(v) for s in spans for v in s["attrs"].values()), f"{what} found in span attributes") + check(not any(k.lower().startswith(("http.request.header.authorization", "enduser")) for s in spans + for k in s["attrs"]), "Authorization/enduser attributes in spans") + print("PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id " + "and request_id; no message text, password or token in logs/span attributes") + + +def test_upload_chain(): + user = new_user("chain") + chat = new_chat(user, title="Chain chat") + trace_id, traceparent = obs.new_traceparent() + accepted = upload_traced(user, chat["id"], png(320, 200), traceparent).json + ready = wait_attachment(user, accepted["id"]) + check(ready["status"] == "ready", f"attachment is {ready['status']}") + + def complete(spans): + return by_name(spans, "commit result", "messenger-worker") and by_name(spans, "cleanup staging") + + spans = obs.wait_trace(trace_id, complete, TRACE_WAIT) + print(obs.span_tree(spans)) + server = by_name(spans, "POST /api/v1/chats/{chat_id}/attachments", "messenger-api")[0] + create = by_name(spans, "create attachments", "messenger-api")[0] + publish = by_name(spans, "publish attachments", "messenger-dispatcher")[0] + consume = by_name(spans, "process attachments.process.v1", "messenger-worker")[0] + check(create["kind"] == "PRODUCER" and publish["kind"] == "PRODUCER" and consume["kind"] == "CONSUMER", "kinds") + check(parent_of(spans, create)["span_id"] == server["span_id"], "create is a child of the API request") + check(publish["parent_id"] == create["span_id"], "publish (dispatcher) is a child of create (API outbox)") + check(consume["parent_id"] == publish["span_id"], "consume (worker) is a child of publish (W3C via AMQP headers)") + check(consume["attrs"].get("messenger.job.outcome") == "ready", "consumer outcome ready") + puts = [s for s in by_name(spans, "S3 PutObject", "messenger-worker") + if s["attrs"].get("aws.s3.key", "").startswith(f"attachments/{accepted['id']}/v1/")] + check(len(puts) == 4, f"4 PUTs of the final objects, got {len(puts)}") + check({s["service"] for s in spans} >= {"messenger-api", "messenger-dispatcher", "messenger-worker"}, "services") + events = {labels["event"] for labels, _ in obs.wait_logs( + f'{{service_name="messenger-worker"}} | trace_id="{trace_id}"', 2, LOKI_WAIT)} + check({"job.claimed", "job.ready"} <= events, f"worker logs with this trace_id: {events}") + print(f"worker logs with trace_id {trace_id}: {sorted(events)}") + print(f"PASS: one trace {trace_id}: API 202 -> outbox create -> dispatcher publish -> worker consume -> " + f"4 S3 objects + COMMIT ready ({len(spans)} spans, services: " + f"{', '.join(sorted({s['service'] for s in spans}))})") + + +def test_retry_chain(): + user = new_user("retry") + chat = new_chat(user, title="Retry chat") + before = obs.prom_value('sum(messenger_job_duration_seconds_count{outcome="retry"})', 0.0) + fault("worker_s3_error", 1) + trace_id, traceparent = obs.new_traceparent() + accepted = upload_traced(user, chat["id"], png(64, 48), traceparent).json + ready = wait_attachment(user, accepted["id"]) + check(ready["status"] == "ready", f"attachment is {ready['status']}") + + def complete(spans): + return len(by_name(spans, "process attachments.process.v1")) >= 2 and by_name(spans, "commit result") + + spans = obs.wait_trace(trace_id, complete, TRACE_WAIT) + print(obs.span_tree(spans)) + attempts = sorted(by_name(spans, "process attachments.process.v1", "messenger-worker"), key=lambda s: s["start"]) + first, second = attempts[0], attempts[-1] + check(first["status"] == "ERROR" and first["attrs"].get("messenger.job.outcome") == "retry", "attempt 1 = retry") + check(first["attrs"].get("error.type") == "storage_unavailable", "attempt 1 error.type") + check(second["attrs"].get("messenger.job.outcome") == "ready", "attempt 2 = ready") + retry_create = [s for s in by_name(spans, "create attachments") if s["parent_id"] == first["span_id"]] + check(retry_create, "retry message is created inside attempt 1 (outbox in the same transaction)") + second_publish = parent_of(spans, second) + check(second_publish["name"] == "publish attachments" and second_publish["parent_id"] == retry_create[0]["span_id"], + "attempt 2 consumes the retry publication") + check(len(by_name(spans, "publish attachments")) == 2, "two publications in one trace") + rows = obs.wait_logs(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="job.retry"', 1, LOKI_WAIT) + print(f"worker log: {rows[0][1]!r} error_type={rows[0][0].get('error_type')}") + after = obs.wait_metric('sum(messenger_job_duration_seconds_count{outcome="retry"})', lambda v: v > before, 40) + print(f"messenger_job_duration_seconds_count{{outcome=\"retry\"}}: {before:g} -> {after:g}") + print(f"PASS: retry in one trace {trace_id}: attempt 1 ERROR storage_unavailable -> create -> publish -> " + f"attempt 2 ready; retry counted in metrics and logged") + + +def test_replay_link(): + user = new_user("replay") + chat = new_chat(user, title="Replay chat") + fault("worker_s3_error", 5) + trace_id, traceparent = obs.new_traceparent() + accepted = upload_traced(user, chat["id"], png(32, 32), traceparent).json + failed = wait_attachment(user, accepted["id"], statuses=("failed",), seconds=120) + check(failed["error_code"] == "retry_exhausted", f"error_code {failed['error_code']}") + job = next(j for j in ops("failed", "--limit", "20") if j["attachment_id"] == accepted["id"]) + result = ops("replay", job["job_id"], "--reason", "lab6 link test") + check("replayed" in result, f"replay refused: {result}") + ready = wait_attachment(user, accepted["id"]) + check(ready["status"] == "ready", "ready after replay") + original = obs.wait_trace(trace_id, lambda s: len(by_name(s, "process attachments.process.v1")) >= 5, TRACE_WAIT) + print(f"original trace {trace_id}: {len(by_name(original, 'process attachments.process.v1'))} attempts, " + f"last outcome {sorted(by_name(original, 'process attachments.process.v1'), key=lambda s: s['start'])[-1]['attrs'].get('messenger.job.outcome')}") + found = None + deadline = time.monotonic() + TRACE_WAIT + while found is None and time.monotonic() < deadline: + for item in obs.tempo_search(f'{{ name = "create attachments" && span.messenger.job.id = "{job["job_id"]}" }}', + minutes=15, limit=20): + if item["traceID"].rjust(32, "0") != trace_id: + found = item["traceID"].rjust(32, "0") + if found is None: + time.sleep(2) + check(found, "replay trace not found") + spans = obs.wait_trace(found, lambda s: by_name(s, "commit result"), TRACE_WAIT) + print(obs.span_tree(spans)) + create = by_name(spans, "create attachments")[0] + linked = {link["trace_id"] for link in create["links"]} + check(trace_id in linked, f"create span of the replay links to the original trace {trace_id}: {linked}") + print(f"replay trace {found}: create span links -> {sorted(linked)} (original upload trace {trace_id})") + check(by_name(spans, "ops replay"), "ops replay span is the root of the new trace") + check(by_name(spans, "process attachments.process.v1")[0]["attrs"].get("messenger.job.outcome") == "ready", + "replayed attempt ready") + print("PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it") + + +def test_history(): + user = new_user("history") + chat = new_chat(user, title="History chat") + trace_id, traceparent = obs.new_traceparent() + call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": "history"}, + headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}) + obs.wait_trace(trace_id, seconds=TRACE_WAIT) + obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"', 1, LOKI_WAIT) + query = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})' + stamp = round(time.time() - 30) # значение на момент t-30 с (параметр time запроса API Prometheus) + rows = obs.prom_query(query, at=stamp) + before = rows[0][1] if rows else None + check(before, "metric sample before restart") + print(f"before restart: trace {trace_id} found, its log found, {query} at t-30s = {before:g}") + compose("restart", "prometheus", "loki", "tempo") + deadline = time.monotonic() + 120 + while True: + try: + spans = obs.tempo_trace(trace_id) + logs = obs.loki_query(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"') + rows = obs.prom_query(query, at=stamp) + after = rows[0][1] if rows else None + if spans and logs and after == before: + break + except OSError: + pass + check(time.monotonic() < deadline, "history not available 120 s after restart") + time.sleep(3) + print(f"after restart of prometheus, loki, tempo: trace {trace_id} ({len(spans)} spans), " + f"{len(logs)} log record(s), metric at t-30s = {after:g} (same value)") + print("PASS: persistent volumes keep traces, logs and metrics across restarts of the stores") + + +SCENARIOS = {"request": test_request, "upload-chain": test_upload_chain, "retry-chain": test_retry_chain, + "replay-link": test_replay_link, "history": test_history} + +if __name__ == "__main__": + names = sys.argv[1:] or list(SCENARIOS) + unknown = [n for n in names if n not in SCENARIOS] + if unknown: + raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}") + run_tests([SCENARIOS[n] for n in names]) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..01f1ced --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,222 @@ +"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик. + +Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что +тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково +(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше +DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование) +при тех же 10 загрузках не помещается в лимит 512 MiB. + +Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков +одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status +(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера +(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с. + + python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json] +""" +import argparse +import hashlib +import json +import platform +import statistics +import threading +import time + +import images +from common import (ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_attachment, + wait_balanced, wait_ready) + + +def proc_status(container): + """VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты).""" + script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; " + "for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current " + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; " + "do [ -r $f ] && echo \"$f $(cat $f)\"; done; true") + out = docker("exec", container, "sh", "-c", script) + values = {} + for line in out.splitlines(): + parts = line.replace(":", " ").split() + if parts[0] in ("VmRSS", "VmHWM"): + values[parts[0]] = int(parts[1]) + elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")): + values["cgroup_peak"] = int(parts[1]) + elif parts[0].endswith(("memory.current", "usage_in_bytes")): + values["cgroup_current"] = int(parts[1]) + return values + + +class Sampler(threading.Thread): + def __init__(self, containers): + super().__init__(daemon=True) + self.containers = containers + self.stop = threading.Event() + self.rss = {c: 0 for c in containers} + self.cgroup = {c: 0 for c in containers} + self.samples = 0 + + def run(self): + while not self.stop.is_set(): + for c in self.containers: + try: + v = proc_status(c) + except Exception: # noqa: BLE001 — один пропущенный замер не важен + continue + self.rss[c] = max(self.rss[c], v.get("VmRSS", 0)) + self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0)) + self.samples += 1 + + +def api_containers(): + from common import api_containers as running + return running() + + +def fresh_replicas(): + compose("restart", "api") + containers = wait_balanced(2, seconds=90) + wait_ready() + return containers + + +def warm_up(user, chat): + """По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п.""" + for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4: + upload(user, chat["id"], data, ctype) + + +def scenario(name, payloads, concurrency): + user = new_user("mem") + chat = new_chat(user) + containers = fresh_replicas() + warm_up(user, chat) + before = {c: proc_status(c) for c in containers} + sampler = Sampler(containers) + sampler.start() + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(i): + data, ctype = payloads[i % len(payloads)] + barrier.wait() + started = time.monotonic() + try: + # lab5: замеряем только приём (202 queued); обработку worker'ом ждём ниже, вне замера + reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180, wait=False) + results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data) + except AssertionError as exc: + results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] + wall = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - wall + time.sleep(0.5) + sampler.stop.set() + sampler.join() + after = {c: proc_status(c) for c in containers} + + ok = [r for r in results if r[0] == 202] + check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 202]}") + for status, _, _, att, data in ok[:3]: + att = wait_attachment(user, att["id"], seconds=120) + check(att["status"] == "ready", f"{name}: attachment {att['id']} is {att['status']}, expected ready") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"]) + check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs") + latencies = sorted(r[1] for r in results) + per_replica = [] + for c in containers: + b, a = before[c], after[c] + per_replica.append({ + "replica": c[:12], + "uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])), + "rss_before_mib": round(b["VmRSS"] / 1024, 1), + "rss_peak_mib": round(a["VmHWM"] / 1024, 1), + "rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1), + "rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1), + "cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None, + "cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None, + }) + sizes = sorted({len(d) for d, _ in payloads}) + summary = { + "scenario": name, + "concurrency": concurrency, + "file_bytes": sizes, + "total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1), + "all_202": True, + "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)}, + "sampler_rounds": sampler.samples, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False)) + return summary + + +# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование. +NAIVE_VS_STREAMING = r''' +import json, os, subprocess, sys +code = r""" +import io, os, resource, struct, sys, zlib +sys.path.insert(0, os.getcwd()) +os.environ.setdefault("S3_ACCESS_KEY_ID", "x") +from PIL import Image +import uploads # импорт до замера: сравниваем только саму проверку файла +Image.MAX_IMAGE_PIXELS = None +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +w, h = 8000, 5000 +comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3) +idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush() +data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") +del idat, comp, row +base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if sys.argv[1] == "naive": + body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB) + im = Image.open(io.BytesIO(data)); im.load() +else: + f = io.BytesIO(data); uploads.check_png(f) +print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024) +""" +out = {} +for mode in ("naive", "streaming"): + r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd()) + out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:] +print(json.dumps(out)) +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload-memory.json") + args = parser.parse_args() + check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads") + + print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}") + print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...") + large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)] + pixels = [(images.png(8000, 5000), "image/png")] + jpeg = [(images.sample_jpeg(), "image/jpeg")] + print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes") + + runs = [ + scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency), + scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency), + scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency), + ] + naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1]) + print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) " + f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB") + limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split() + result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20, + "api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive} + (ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"]) + print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB " + f"(limit {result['api_limit_mib']} MiB); results in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/worker_probe.py b/student-tests/worker_probe.py new file mode 100644 index 0000000..7b75584 --- /dev/null +++ b/student-tests/worker_probe.py @@ -0,0 +1,148 @@ +"""Пробник, который выполняется ВНУТРИ контейнера worker (там Pillow, модули приложения, доступ к БД и S3). + +Тест передаёт этот файл через stdin: `docker compose exec -T worker python - <команда> [аргументы]`. +Секреты остаются в контейнере; наружу выходит одна строка JSON (без ключей доступа, только факты). + + make — тестовое изображение, base64 (kind: thumb320, exif6, alpha, tiny, large, strip, big40) + inspect — метаданные из БД + декодированные объекты из S3: размеры, формат, EXIF, + цвета контрольных точек, SHA-256, список ключей под attachments// + exists — есть ли объект в bucket + imaging-limits — imaging.py на 40-Мп PNG: пик RSS дочернего процесса и отказ по лимиту пикселей +""" +import base64 +import hashlib +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +from uuid import UUID + +from PIL import Image + +import psycopg +from psycopg.rows import dict_row + +import settings +import storage + + +def make(kind: str) -> bytes: + out = io.BytesIO() + if kind == "thumb320": # 320x200 -> thumbnail 256x160 (контракт) + img = Image.new("RGB", (320, 200), (200, 30, 30)) + img.paste((30, 30, 200), (0, 0, 160, 200)) + img.save(out, "PNG") + elif kind == "exif6": # JPEG 400x300 с EXIF Orientation=6 и Make/Model/GPS + img = Image.new("RGB", (400, 300), (0, 160, 0)) + img.paste((255, 0, 0), (0, 0, 100, 300)) # левая полоса красная: после поворота на 90° по часовой — сверху + exif = Image.Exif() + exif[0x0112] = 6 + exif[0x010F] = "ProbeCamera" + exif[0x0110] = "Model-X" + exif[0x8825] = {1: "N", 2: (55.0, 45.0, 0.0)} + img.save(out, "JPEG", quality=95, exif=exif.tobytes()) + elif kind == "alpha": # RGBA 300x200: левая половина прозрачная + img = Image.new("RGBA", (300, 200), (0, 0, 255, 255)) + img.paste((0, 0, 0, 0), (0, 0, 150, 200)) + img.save(out, "PNG") + elif kind == "tiny": # 40x30: без увеличения для thumbnail/optimized + Image.new("RGB", (40, 30), (10, 120, 220)).save(out, "PNG") + elif kind == "large": # 3000x2000 JPEG: optimized 1280x853, thumbnail 256x170 + img = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB") + img.save(out, "JPEG", quality=90) + elif kind == "strip": # 5000x2: thumbnail 256x1 (минимум 1 px) + Image.new("RGB", (5000, 2), (90, 90, 90)).save(out, "PNG") + elif kind == "big40": # 8000x5000 = 40 Мп ровно на лимите + Image.new("RGB", (8000, 5000), (120, 60, 30)).save(out, "PNG") + else: + raise SystemExit("unknown kind") + return out.getvalue() + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row) + + +def read(key: str) -> bytes: + stream = storage.open_read(key) + try: + return b"".join(storage.iter_chunks(stream)) + finally: + stream.close() + + +def inspect(attachment_id: str) -> dict: + with connect() as conn: + a = conn.execute("SELECT state, object_key, source_key, pipeline_version, sha256, size_bytes " + "FROM attachments WHERE id = %s", (UUID(attachment_id),)).fetchone() + variants = conn.execute("SELECT name, content_type, width, height, sha256, size_bytes, object_key " + "FROM attachment_variants WHERE attachment_id = %s ORDER BY name", + (UUID(attachment_id),)).fetchall() + result = {"state": a["state"], "staging_present": a["source_key"] is not None and storage.exists(a["source_key"]), + "variant_rows": len(variants), "variants": {}} + if a["object_key"]: + original = read(a["object_key"]) + result["original_sha_ok"] = hashlib.sha256(original).hexdigest() == a["sha256"] + result["original_size_ok"] = len(original) == a["size_bytes"] + for v in variants: + data = read(v["object_key"]) + img = Image.open(io.BytesIO(data)) + info = {"format": img.format, "size": list(img.size), "mode": img.mode, + "meta_size_ok": [v["width"], v["height"]] == list(img.size), + "sha_ok": hashlib.sha256(data).hexdigest() == v["sha256"] and len(data) == v["size_bytes"], + "exif_tags": len(img.getexif()), "has_xmp": "xmp" in img.info or b"http://ns.adobe.com/xap" in data, + "quality_hint": None} + rgb = img.convert("RGBA") + w, h = rgb.size + info["pixel_top_center"] = list(rgb.getpixel((w // 2, 0))) + info["pixel_left_center"] = list(rgb.getpixel((0, h // 2))) + info["pixel_right_center"] = list(rgb.getpixel((w - 1, h // 2))) + result["variants"][v["name"]] = info + prefix = f"attachments/{attachment_id}/" + result["final_keys"] = sorted(key[len(prefix):] for key, _m, _s in storage.list_objects(prefix)) + return result + + +def imaging_limits() -> dict: + """Пик памяти процесса обработки на худшем допустимом входе и отказ при превышении лимита пикселей.""" + here = os.path.dirname(os.path.abspath(storage.__file__)) + report = {} + with tempfile.TemporaryDirectory(dir=settings.WORKER_TMP_DIR) as tmp: + source = os.path.join(tmp, "big.png") + with open(source, "wb") as f: + f.write(make("big40")) + env = {"PATH": os.environ.get("PATH", ""), "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for label, max_pixels in (("at_limit_40mpx", 40_000_000), ("over_limit_max_10mpx", 10_000_000)): + out = os.path.join(tmp, label) + os.mkdir(out) + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + proc = subprocess.run([sys.executable, os.path.join(here, "imaging.py"), source, out, "image/png", + str(max_pixels)], env=env, capture_output=True, timeout=120) + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + report[label] = {"exit_code": proc.returncode, "peak_rss_mib_max_children": round(peak / 1024, 1), + "rss_grew": peak > before} + report["memory_limit_mib"] = settings.WORKER_MEMORY_LIMIT_MB + report["timeout_s"] = settings.JOB_TIMEOUT_SECONDS + return report + + +def main(): + command = sys.argv[1] + if command == "make": + print(json.dumps({"b64": base64.b64encode(make(sys.argv[2])).decode("ascii")})) + elif command == "inspect": + print(json.dumps(inspect(sys.argv[2]))) + elif command == "exists": + print(json.dumps({"exists": storage.exists(sys.argv[2])})) + elif command == "imaging-limits": + print(json.dumps(imaging_limits())) + else: + raise SystemExit("unknown command") + + +if __name__ == "__main__": + main() diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..686f3f9 --- /dev/null +++ b/telemetry.py @@ -0,0 +1,450 @@ +"""Наблюдаемость (lab6): OpenTelemetry SDK для api, worker, dispatcher, janitor и ops. + +Все три сигнала уходят по OTLP/HTTP (protobuf) в OTel Collector (OTEL_EXPORTER_OTLP_ENDPOINT): + traces -> Collector -> Tempo + metrics -> Collector -> Prometheus (встроенный OTLP receiver) + logs -> Collector -> Loki (OTLP endpoint); те же записи пишутся в stderr одной JSON-строкой + (docker compose logs работает и без стека наблюдаемости). + +Ресурс (одинаковый у всех сигналов процесса): service.name (messenger-api / messenger-worker / +messenger-dispatcher / messenger-janitor), service.version (APP_VERSION), service.instance.id +(hostname контейнера = X-Instance-Id), deployment.environment.name=lab (OTEL_RESOURCE_ATTRIBUTES). + +Память при недоступном Collector или хранилище ограничена (экспорт идёт в фоновых потоках SDK, +запрос пользователя его не ждёт): + - спаны: BatchSpanProcessor, очередь OTEL_BSP_MAX_QUEUE_SIZE (2048); лишние отбрасываются и считаются + в otel.sdk.processor.span.processed{error.type="queue_full"} (OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED); + - логи: BatchLogRecordProcessor, очередь OTEL_BLRP_MAX_QUEUE_SIZE (2048), так же; + - метрики: агрегаты фиксированного размера (метки низкой кардинальности), экспорт раз в + OTEL_METRIC_EXPORT_INTERVAL; неудачный экспорт не накапливается, cumulative-счётчики догонят; + - каждый экспорт ограничен OTEL_EXPORTER_OTLP_TIMEOUT (5 с). + +Контекст между процессами — только W3C Trace Context (traceparent/tracestate); baggage не +распространяется, чтобы через заголовки не утекали произвольные данные. + +Чего НЕТ в телеметрии: паролей, Authorization, токенов сессий и их хешей, текстов сообщений, имён +файлов, байтов изображений, ключей и подписей S3. Метки метрик — только низкокардинальные: метод, +шаблон маршрута, класс/код статуса, тип и исход задания, имя очереди, состояние соединения. +В спанах и логах есть технические ID синтетических данных (job_id, attachment_id, request_id): +без них нельзя пройти от загрузки к результату worker; ID пользователя/чата туда не пишутся. +""" +import json +import logging +import os +import sys +import threading +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +from opentelemetry import context as otel_context +from opentelemetry import metrics, propagate, trace +from opentelemetry.trace import Link, SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +import settings + +SERVICE = "messenger" +VERSION = settings.APP_VERSION +_propagator = TraceContextTextMapPropagator() +propagate.set_global_textmap(_propagator) + +tracer = trace.get_tracer("messenger", VERSION) +meter = metrics.get_meter("messenger", VERSION) + +_state = {"configured": False, "providers": []} + +# Границы buckets (секунды) под ожидаемые задержки: p50/p95/p99 считаются из buckets в Prometheus. +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10) +JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120) +READY_BUCKETS = (0.5, 1, 2, 5, 10, 15, 30, 60, 120, 300, 600) +POOL_WAIT_BUCKETS = (0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5) + + +def enabled() -> bool: + return os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true" + + +# ---------- логи: JSON в stderr + OTLP ---------- + +_RESERVED = set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message", "asctime", "taskName"} +# Ключи, которые никогда не уходят ни в stderr, ни в OTLP (защита от случайного extra={...}). +_FORBIDDEN_KEYS = {"password", "authorization", "token", "secret", "text", "body", "filename"} + + +def _extra(record: logging.LogRecord) -> dict: + out = {} + for key, value in vars(record).items(): + if key in _RESERVED or key.startswith("_") or key.lower() in _FORBIDDEN_KEYS: + continue + if isinstance(value, (str, bool, int, float)): + out[key] = value + elif value is not None: + out[key] = str(value) + return out + + +class _EventFilter(logging.Filter): + """У каждой записи есть поле event (контракт): явное extra={"event": ...} или имя логгера.""" + + def filter(self, record): + if not getattr(record, "event", None): + record.event = record.name + return True + + +class _RateLimit(logging.Filter): + """Ошибки экспорта самого SDK (Collector недоступен) — не чаще раза в 30 с на источник.""" + + def __init__(self, seconds=30.0): + super().__init__() + self.seconds = seconds + self.last = {} + self.lock = threading.Lock() + + def filter(self, record): + if not record.name.startswith("opentelemetry"): + return True + key = (record.name, record.levelno) + now = time.monotonic() + with self.lock: + if now - self.last.get(key, -1e9) < self.seconds: + return False + self.last[key] = now + return True + + +class JsonFormatter(logging.Formatter): + """Одна JSON-строка: ts, severity, service.name, event, message, trace_id/span_id (если есть спан).""" + + def __init__(self, service_name: str): + super().__init__() + self.service_name = service_name + + def format(self, record): + data = { + "ts": datetime.fromtimestamp(record.created, timezone.utc).isoformat(timespec="milliseconds"), + "severity": record.levelname, + "service.name": self.service_name, + "event": getattr(record, "event", record.name), + "logger": record.name, + "message": record.getMessage(), + } + span = trace.get_current_span().get_span_context() + if span.is_valid: + data["trace_id"] = format(span.trace_id, "032x") + data["span_id"] = format(span.span_id, "016x") + data.update({k: v for k, v in _extra(record).items() if k != "event"}) + if record.exc_info and record.exc_info[0] is not None: + data["exception.type"] = record.exc_info[0].__name__ + data["exception.stacktrace"] = self.formatException(record.exc_info) + return json.dumps(data, ensure_ascii=False, default=str) + + +def _severity_map(): + from opentelemetry._logs import SeverityNumber + return {"DEBUG": SeverityNumber.DEBUG, "INFO": SeverityNumber.INFO, "WARNING": SeverityNumber.WARN, + "ERROR": SeverityNumber.ERROR, "CRITICAL": SeverityNumber.FATAL} + + +_SEVERITY = _severity_map() + + +class OtlpLogHandler(logging.Handler): + """logging -> OpenTelemetry Logs API. trace_id/span_id берутся из текущего контекста автоматически.""" + + def __init__(self, logger_provider): + super().__init__(level=logging.INFO) + self.provider = logger_provider + self.loggers = {} + + def emit(self, record): + if record.name.startswith("opentelemetry"): + return # ошибки экспорта не отправляем тем же экспортом (иначе петля при недоступном Collector) + try: + from opentelemetry._logs import LogRecord, SeverityNumber + + logger = self.loggers.get(record.name) + if logger is None: + logger = self.loggers[record.name] = self.provider.get_logger(record.name) + attributes = _extra(record) + attributes["code.function.name"] = record.funcName + attributes["code.line.number"] = record.lineno + if record.exc_info and record.exc_info[0] is not None: + attributes["exception.type"] = record.exc_info[0].__name__ + attributes["exception.stacktrace"] = logging.Formatter().formatException(record.exc_info) + level = {"WARNING": "WARN", "CRITICAL": "FATAL"}.get(record.levelname, record.levelname) + logger.emit(LogRecord( + timestamp=int(record.created * 1e9), + severity_text=level, + severity_number=_SEVERITY.get(record.levelname, SeverityNumber.INFO), + body=record.getMessage(), + attributes=attributes, + event_name=attributes.get("event"), + )) + except Exception: # noqa: BLE001 — сбой телеметрии не должен ломать логирование и запрос + self.handleError(record) + + +def _setup_logging(service_name: str, logger_provider=None) -> None: + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + root.setLevel(logging.INFO) + # stderr, как logging.basicConfig в lab5: stdout CLI (janitor --once, ops.py) остаётся чистым JSON-отчётом. + console = logging.StreamHandler(sys.stderr) + console.setFormatter(JsonFormatter(service_name)) + console.addFilter(_EventFilter()) + console.addFilter(_RateLimit()) + root.addHandler(console) + if logger_provider is not None: + otlp = OtlpLogHandler(logger_provider) + otlp.addFilter(_EventFilter()) + root.addHandler(otlp) + logging.getLogger("pika").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("opentelemetry").setLevel(logging.WARNING) + + +# ---------- сэмплирование ---------- + +def _sampler(): + """ParentBased(AlwaysOn), но клиентский спан БЕЗ родителя не записывается. + + Такие спаны — фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, janitor): + отдельная трасса на каждый SELECT только забивала бы Tempo. Всё, что относится к запросу или + заданию, имеет родителя (server/consumer/producer span) и сохраняется полностью. + """ + from opentelemetry.sdk.trace.sampling import (ALWAYS_ON, Decision, ParentBased, Sampler, + SamplingResult) + + class DropOrphanClientSpans(Sampler): + def __init__(self): + self.inner = ParentBased(ALWAYS_ON) + + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + parent = trace.get_current_span(parent_context).get_span_context() + if not parent.is_valid and kind == SpanKind.CLIENT: + return SamplingResult(Decision.DROP) + return self.inner.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state) + + def get_description(self): + return "ParentBased(AlwaysOn) without orphan client spans" + + return DropOrphanClientSpans() + + +# ---------- process / container ---------- + +def _read(path: str) -> str | None: + try: + return Path(path).read_text(encoding="ascii").strip() + except OSError: + return None + + +def _register_process_metrics() -> None: + from opentelemetry.metrics import CallbackOptions, Observation + + page = os.sysconf("SC_PAGE_SIZE") + + def cpu(_options: CallbackOptions): + times = os.times() + return [Observation(times.user, {"cpu.mode": "user"}), Observation(times.system, {"cpu.mode": "system"})] + + def memory(_options: CallbackOptions): + statm = _read("/proc/self/statm") + return [Observation(int(statm.split()[1]) * page)] if statm else [] + + meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU time of the process") + meter.create_observable_up_down_counter("process.memory.usage", [memory], unit="By", + description="Resident set size of the process") + + # cgroup v2 (Docker Desktop, современный Linux): контейнер целиком, включая дочерние процессы + # (у worker это imaging.py). Вне контейнера/на cgroup v1 этих метрик просто нет. + if _read("/sys/fs/cgroup/memory.current") is None: + return + + def container_memory(_options: CallbackOptions): + value = _read("/sys/fs/cgroup/memory.current") + return [Observation(int(value))] if value else [] + + def container_memory_limit(_options: CallbackOptions): + value = _read("/sys/fs/cgroup/memory.max") + return [Observation(int(value))] if value and value != "max" else [] + + def container_cpu(_options: CallbackOptions): + stat = _read("/sys/fs/cgroup/cpu.stat") or "" + for line in stat.splitlines(): + name, _, value = line.partition(" ") + if name == "usage_usec": + return [Observation(int(value) / 1e6)] + return [] + + def container_cpu_limit(_options: CallbackOptions): + value = (_read("/sys/fs/cgroup/cpu.max") or "").split() + if len(value) == 2 and value[0] != "max": + return [Observation(int(value[0]) / int(value[1]))] + return [] + + meter.create_observable_up_down_counter("container.memory.usage", [container_memory], unit="By", + description="Memory of the container cgroup") + meter.create_observable_up_down_counter("container.memory.limit", [container_memory_limit], unit="By", + description="Memory limit of the container cgroup") + meter.create_observable_counter("container.cpu.time", [container_cpu], unit="s", + description="CPU time of the container cgroup") + meter.create_observable_up_down_counter("container.cpu.limit", [container_cpu_limit], unit="{cpu}", + description="CPU limit of the container (cores)") + + +# ---------- настройка ---------- + +def setup(service_name: str, *, instrument_db: bool = True) -> None: + """Вызывается один раз при старте процесса, ДО открытия соединений с БД.""" + if _state["configured"]: + return + _state["configured"] = True + if not enabled(): + _setup_logging(service_name) + logging.getLogger("messenger.telemetry").info("OpenTelemetry disabled (OTEL_SDK_DISABLED=true)", + extra={"event": "telemetry.disabled"}) + return + + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab) читает Resource.create; явные атрибуты + # важнее переменных окружения: janitor и ops.py с общим .env не станут «messenger-api». + resource = Resource.create({ + "service.name": service_name, + "service.version": VERSION, + "service.instance.id": settings.INSTANCE_ID, + }) + + views = [ + View(instrument_name="http.server.request.duration", + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + View(instrument_name="db.client.connection.wait_time", + aggregation=ExplicitBucketHistogramAggregation(POOL_WAIT_BUCKETS)), + ] + interval = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "10000")) + meter_provider = MeterProvider( + resource=resource, views=views, + metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(), export_interval_millis=interval)], + ) + metrics.set_meter_provider(meter_provider) + + tracer_provider = TracerProvider(resource=resource, sampler=_sampler()) + tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(), meter_provider=meter_provider)) + trace.set_tracer_provider(tracer_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(), + meter_provider=meter_provider)) + set_logger_provider(logger_provider) + _state["providers"] = [tracer_provider, logger_provider, meter_provider] + + _setup_logging(service_name, logger_provider) + _register_process_metrics() + if instrument_db: + # Спаны PostgreSQL: один на запрос, текст SQL с плейсхолдерами; значения параметров НЕ пишутся + # (capture_parameters=False), sqlcommenter выключен. + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + PsycopgInstrumentor().instrument(tracer_provider=tracer_provider, enable_commenter=False, + capture_parameters=False, skip_dep_check=True) + logging.getLogger("messenger.telemetry").info( + "OpenTelemetry enabled: OTLP %s, service %s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "default"), + service_name, extra={"event": "telemetry.enabled"}) + + +def shutdown() -> None: + """Дослать буферы при штатной остановке (не дольше таймаута экспорта).""" + for provider in _state["providers"]: + try: + provider.shutdown() + except Exception: # noqa: BLE001 + pass + _state["providers"] = [] + + +# ---------- контекст W3C ---------- + +def inject(carrier: dict | None = None) -> dict: + """traceparent/tracestate текущего спана в carrier (заголовки AMQP, поле сообщения).""" + carrier = {} if carrier is None else carrier + _propagator.inject(carrier) + return carrier + + +def extract(carrier) -> otel_context.Context: + """Контекст из входящих заголовков (HTTP, AMQP headers, поле traceparent сообщения).""" + return _propagator.extract(carrier or {}) + + +def current_traceparent() -> str | None: + return inject().get("traceparent") + + +def context_from(traceparent: str | None, tracestate: str | None = None) -> otel_context.Context: + carrier = {"traceparent": traceparent} if traceparent else {} + if traceparent and tracestate: + carrier["tracestate"] = tracestate + return extract(carrier) + + +def link_to(traceparent: str | None) -> list: + """Link на другой контекст (например, исходная загрузка при replay оператора).""" + span_context = trace.get_current_span(context_from(traceparent)).get_span_context() + return [Link(span_context)] if span_context.is_valid else [] + + +def trace_id_hex() -> str | None: + span = trace.get_current_span().get_span_context() + return format(span.trace_id, "032x") if span.is_valid else None + + +# ---------- помощники для спанов ---------- + +def mark_error(span, error_type: str, message: str | None = None) -> None: + """Статус ERROR + error.type (низкокардинальный код). Текст — только безопасный код/класс.""" + span.set_attribute("error.type", error_type) + span.set_status(Status(StatusCode.ERROR, message or error_type)) + + +@contextmanager +def client_span(name: str, attributes: dict | None = None): + """Клиентский спан (S3, Redis). Исключение -> ERROR с типом исключения; само исключение не глотается.""" + with tracer.start_as_current_span(name, kind=SpanKind.CLIENT, attributes=attributes or {}, + record_exception=False, set_status_on_exception=False) as span: + try: + yield span + except Exception as exc: + mark_error(span, type(exc).__name__) + raise + + +@contextmanager +def internal_span(name: str, attributes: dict | None = None): + with tracer.start_as_current_span(name, kind=SpanKind.INTERNAL, attributes=attributes or {}, + record_exception=False, set_status_on_exception=False) as span: + try: + yield span + except Exception as exc: + mark_error(span, type(exc).__name__) + raise diff --git a/uploads.py b/uploads.py new file mode 100644 index 0000000..c01bfc1 --- /dev/null +++ b/uploads.py @@ -0,0 +1,519 @@ +"""Приём и проверка загружаемого изображения с ограниченной памятью. + +1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше + MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном + файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. +2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях + читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково + (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), + JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. +""" +import asyncio +import hashlib +import logging +import struct +import tempfile +import threading +import time +import unicodedata +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image +from starlette.requests import ClientDisconnect, Request + +import settings +from errors import bad_request, payload_too_large, unsupported_media_type + +log = logging.getLogger("messenger.uploads") + +PNG = "image/png" +JPEG = "image/jpeg" +ALLOWED_TYPES = (PNG, JPEG) +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + +SPOOL_IN_MEMORY = 256 * 1024 +INFLATE_CHUNK = 64 * 1024 +CRC_CHUNK = 64 * 1024 + +# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). +Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS + +# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. +_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) + + +@dataclass +class Upload: + file: BinaryIO + size: int + sha256: str + + +@dataclass +class ImageInfo: + content_type: str + width: int + height: int + + +class Damaged(Exception): + pass + + +class TooManyPixels(Exception): + pass + + +# ---------- имя файла и заголовки ---------- + +def validate_filename(value: str | None) -> str: + """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" + if value is None: + raise bad_request("Query parameter filename is required") + if not 1 <= len(value) <= 255 or not value.strip(): + raise bad_request("filename must be 1-255 characters") + if value in (".", ".."): + raise bad_request("filename is not allowed") + for char in value: + if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): + raise bad_request("filename contains forbidden characters") + return value + + +def declared_type(header: str | None) -> str: + base = (header or "").split(";", 1)[0].strip().lower() + if base not in ALLOWED_TYPES: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return base + + +def declared_length(header: str | None) -> int | None: + if header is None: + return None + if not header.isdigit(): + raise bad_request("Invalid Content-Length") + length = int(header) + if length > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + return length + + +# ---------- приём тела ---------- + +async def receive(request: Request) -> Upload: + """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" + spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS + stream = request.stream().__aiter__() + try: + while True: + wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) + if wait <= 0: + raise bad_request("Upload took too long") + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + raise bad_request("Upload stalled or took too long") from None + if not chunk: + continue + size += len(chunk) + if size > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + digest.update(chunk) + spool.write(chunk) + except ClientDisconnect: + spool.close() + log.info("Upload aborted by the client after %s bytes, nothing stored", size) + raise bad_request("Upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad_request("Empty body") + return Upload(spool, size, digest.hexdigest()) + + +async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: + """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. + + Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. + """ + read = 0 + deadline = time.monotonic() + seconds + try: + stream = request.stream().__aiter__() + while read <= limit: + wait = deadline - time.monotonic() + if wait <= 0: + return + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except (StopAsyncIteration, asyncio.TimeoutError): + return + read += len(chunk) + except (ClientDisconnect, RuntimeError): + return + + +# ---------- проверка содержимого ---------- + +def sniff(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return PNG + if head.startswith(JPEG_SIGNATURE): + return JPEG + return None + + +def inspect(upload: Upload, declared: str) -> ImageInfo: + upload.file.seek(0) + detected = sniff(upload.file.read(16)) + if detected is None: + raise unsupported_media_type("File is not a PNG or JPEG image") + if detected != declared: + raise unsupported_media_type("Content-Type does not match the file content") + upload.file.seek(0) + try: + with _decode_slots: + if detected == PNG: + width, height = check_png(upload.file) + upload.file.seek(0) + check_png_metadata(upload.file) + else: + width, height = check_jpeg(upload.file) + except TooManyPixels: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except Damaged as exc: + log.info("Rejected damaged %s: %s", detected, exc) + raise unsupported_media_type("Image is damaged or not supported") from None + except Image.DecompressionBombError: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: + log.info("Rejected damaged %s: %s", detected, type(exc).__name__) + raise unsupported_media_type("Image is damaged or not supported") from None + finally: + upload.file.seek(0) + return ImageInfo(detected, width, height) + + +def check_pixels(width: int, height: int) -> None: + if width < 1 or height < 1: + raise Damaged("zero dimension") + if width * height > settings.MAX_IMAGE_PIXELS: + raise TooManyPixels() + + +def check_jpeg(file: BinaryIO) -> tuple[int, int]: + with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS + width, height = image.size + check_pixels(width, height) + # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: + # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + return width, height + + +# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +FILTER_BYTES = bytes(range(5)) + + +class PngRows: + """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" + + def __init__(self, width, height, depth, color, interlace): + bits = depth * PNG_CHANNELS[color] + + def row_bytes(w): + return (w * bits + 7) // 8 + + passes = [(width, height)] if interlace == 0 else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + # (длина строки с байтом фильтра, байт в проходе) + self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] + self.expected = sum(total for _, total in self.passes) + self.index, self.pos = 0, 0 + self.total = 0 + + def feed(self, data: bytes) -> None: + view = memoryview(data) + self.total += len(data) + if self.total > self.expected: + raise Damaged("more pixel data than the header says") + offset = 0 + while offset < len(view): + stride, size = self.passes[self.index] + take = min(len(view) - offset, size - self.pos) + first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride + if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): + raise Damaged("invalid filter type") + self.pos += take + offset += take + if self.pos == size: + self.index, self.pos = self.index + 1, 0 + + def done(self) -> bool: + return self.total == self.expected + + +# Чанки, содержимое которых проверяется (держим в памяти; размер ограничен спецификацией): +# IHDR — 13 байт, PLTE — до 256 x 3 = 768, tRNS — до 256 (палитра) / 6 (RGB). +SMALL_CHUNK_LIMIT = {b"IHDR": 13, b"PLTE": 768, b"tRNS": 256} +CRITICAL = (b"IHDR", b"PLTE", b"IDAT", b"IEND") +# После данных изображения допустимы только время и текст (строгий список). +AFTER_IDAT = {b"IEND", b"tIME", b"tEXt", b"zTXt", b"iTXt"} +TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") +# Анимированные PNG (APNG) не поддерживаются: кадры — отдельная сущность, превью строится по одному кадру. +APNG_CHUNKS = (b"acTL", b"fcTL", b"fdAT") +# Как PngImagePlugin.MAX_TEXT_CHUNK в Pillow: больше 1 MiB распакованного текста в одном чанке — отказ. +TEXT_LIMIT = 1024 * 1024 +# Чанки метаданных до IDAT Pillow (Image.open) читает целиком: ограничиваем, чтобы память не росла. +# ICC-профили и EXIF реальных картинок — единицы-сотни KiB. +PRE_IDAT_CHUNK_LIMIT = 2 * 1024 * 1024 +# Чанки, которые по спецификации обязаны стоять ДО первого IDAT (PNG 3rd ed., 5.6 «Chunk ordering»). +# После IDAT допустимы только tIME, tEXt, zTXt, iTXt, кадры APNG и частные чанки. +BEFORE_IDAT = {b"PLTE", b"tRNS", b"cHRM", b"cICP", b"gAMA", b"iCCP", b"mDCV", b"cLLI", b"sBIT", b"sRGB", + b"bKGD", b"hIST", b"pHYs", b"sPLT", b"eXIf"} + + +def check_png(file: BinaryIO) -> tuple[int, int]: + """Структура PNG по спецификации (PNG 3rd ed., гл. 5 и 11) без декодирования пикселей в память. + + Кроме CRC каждого чанка проверяется СМЫСЛ критичных чанков и tRNS: CRC защищает только от + случайной порчи, а намеренно собранный файл с верными CRC может быть некорректным. + """ + if file.read(8) != PNG_SIGNATURE: + raise Damaged("signature") + rows = inflater = None + width = height = depth = 0 + color = None + palette_entries = 0 + seen = set() # какие чанки уже встречались + idat_started = idat_finished = False + first = True + while True: + header = file.read(8) + if len(header) < 8: + raise Damaged("truncated: no IEND") + length, kind = struct.unpack(">I4s", header) + if length > 0x7FFFFFFF: + raise Damaged("chunk length") + if not kind.isalpha() or not kind.isascii(): + raise Damaged("chunk type") + if first and kind != b"IHDR": + raise Damaged("IHDR must be first") + if kind[:1].isupper() and kind not in CRITICAL: + raise Damaged(f"unknown critical chunk {kind!r}") # такой чанк декодер обязан отвергнуть + if kind in SMALL_CHUNK_LIMIT and length > SMALL_CHUNK_LIMIT[kind]: + raise Damaged(f"{kind!r} is too long") + if kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT chunks must be consecutive") + idat_started = True + elif idat_started: + idat_finished = True + if kind in APNG_CHUNKS: + raise Damaged("animated PNG is not supported") + if kind in BEFORE_IDAT and idat_started: + raise Damaged(f"{kind!r} after IDAT") + if idat_finished and kind not in AFTER_IDAT: + raise Damaged(f"{kind!r} after IDAT") + if not idat_started and kind != b"IDAT" and length > PRE_IDAT_CHUNK_LIMIT: + raise Damaged(f"{kind!r} metadata chunk is too large") + if kind == b"tIME" and length != 7: + raise Damaged("tIME size") + if kind in (b"IHDR", b"PLTE", b"tRNS", b"IEND") and kind in seen: + raise Damaged(f"duplicate {kind!r}") + seen.add(kind) + + crc = zlib.crc32(kind) + kept = b"" + text = TextChunk(kind) if kind in TEXT_CHUNKS else None + left = length + while left: + piece = file.read(min(CRC_CHUNK, left)) + if not piece: + raise Damaged("truncated chunk") + left -= len(piece) + crc = zlib.crc32(piece, crc) + if kind in SMALL_CHUNK_LIMIT: + kept += piece + elif kind == b"IDAT": + feed_inflate(inflater, rows, piece) + elif text is not None: + text.feed(piece) + if text is not None: + text.finish() + stored = file.read(4) + if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: + raise Damaged(f"CRC mismatch in {kind!r}") + + if kind == b"IHDR": + if length != 13: + raise Damaged("IHDR") + width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) + if width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise Damaged("dimensions") + check_pixels(width, height) # до распаковки + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: + raise Damaged("color type / bit depth") + if compression != 0 or filter_method != 0 or interlace not in (0, 1): + raise Damaged("IHDR methods") + rows = PngRows(width, height, depth, color, interlace) + inflater = zlib.decompressobj() + elif kind == b"PLTE": + # 1-256 записей по 3 байта; для палитровых — не больше 2^глубина; у серых палитры быть не может. + if color in (0, 4): + raise Damaged("PLTE in a grayscale image") + if length == 0 or length % 3: + raise Damaged("palette size") + palette_entries = length // 3 + if color == 3 and palette_entries > 1 << depth: + raise Damaged("palette larger than the bit depth allows") + elif kind == b"tRNS": + # Прозрачность: серое — 2 байта, RGB — 6, палитра — не больше записей PLTE; при альфа-канале запрещён. + if color in (4, 6): + raise Damaged("tRNS with an alpha channel") + if color == 3 and not palette_entries: + raise Damaged("tRNS before PLTE") + expected_ok = {0: length == 2, 2: length == 6, 3: 1 <= length <= palette_entries}[color] + if not expected_ok: + raise Damaged("tRNS size") + elif kind == b"IEND": + if length != 0: + raise Damaged("IEND with data") + break + first = False + + if file.read(1): + raise Damaged("data after IEND") + if color == 3 and not palette_entries: + raise Damaged("palette image without PLTE") + if not idat_started: + raise Damaged("no IDAT") + tail = inflater.flush(INFLATE_CHUNK) + if tail: + rows.feed(tail) + if not inflater.eof or inflater.unused_data or not rows.done(): + raise Damaged("pixel data is incomplete") + return width, height + + +def check_png_metadata(file: BinaryIO) -> None: + """Второе мнение: стандартный парсер Pillow разбирает метаданные до IDAT (gAMA, iCCP, pHYs, tEXt…), + НЕ читая пиксели. Только open, без verify(): verify читает каждый чанк, включая IDAT до 10 MiB, + целиком в память, а CRC и данные после IDAT уже потоково проверил check_png. + Нужен для вспомогательных чанков: по спецификации их можно игнорировать, но если Pillow + не может их разобрать, файл нельзя будет обработать (превью в lab5) — лучше отклонить сразу. + """ + with Image.open(file, formats=["PNG"]): + pass # open разбирает все чанки до первого IDAT; после IDAT check_png пускает только время и текст + + +class TextChunk: + """Потоковая проверка tEXt / zTXt / iTXt: ключ 1-79 байт, метод сжатия 0, сжатый текст — + корректный zlib не больше TEXT_LIMIT после распаковки (распаковка кусками, текст не хранится).""" + + def __init__(self, kind: bytes): + self.kind = kind + self.head = b"" + self.done_header = False + self.inflater = None + self.size = 0 + + def feed(self, piece: bytes) -> None: + if not self.done_header: + self.head += piece + rest = self._header() + if rest is None: + if len(self.head) > 4096: + raise Damaged("text chunk header") + return + piece = rest + if self.inflater is not None and piece: + self._inflate(piece) + + def _keyword_end(self) -> int | None: + end = self.head.find(b"\x00") + if end == -1: + return None + if not 1 <= end <= 79: + raise Damaged("text keyword") + return end + + def _header(self) -> bytes | None: + end = self._keyword_end() + if end is None: + return None + h = self.head + if self.kind == b"tEXt": + self.done_header = True + return b"" + if self.kind == b"zTXt": + if len(h) < end + 2: + return None + if h[end + 1] != 0: + raise Damaged("zTXt compression method") + self.done_header, self.inflater = True, zlib.decompressobj() + return h[end + 2:] + # iTXt: keyword \0 flag method language \0 translated-keyword \0 text + if len(h) < end + 3: + return None + flag, method = h[end + 1], h[end + 2] + if flag not in (0, 1) or (flag and method != 0): + raise Damaged("iTXt compression") + lang_end = h.find(b"\x00", end + 3) + tkey_end = h.find(b"\x00", lang_end + 1) if lang_end != -1 else -1 + if tkey_end == -1: + return None + self.done_header = True + if flag: + self.inflater = zlib.decompressobj() + return h[tkey_end + 1:] + + def _inflate(self, data: bytes) -> None: + if self.inflater.eof: + raise Damaged("data after compressed text") + out = self.inflater.decompress(data, INFLATE_CHUNK) + self.size += len(out) + while self.inflater.unconsumed_tail and not self.inflater.eof and self.size <= TEXT_LIMIT: + out = self.inflater.decompress(self.inflater.unconsumed_tail, INFLATE_CHUNK) + self.size += len(out) + if self.size > TEXT_LIMIT: + raise Damaged("compressed text is too large") + if self.inflater.eof and self.inflater.unused_data: + raise Damaged("data after compressed text") + + def finish(self) -> None: + if not self.done_header: + raise Damaged("text chunk header") + if self.inflater is not None: + self.size += len(self.inflater.flush(INFLATE_CHUNK)) + if not self.inflater.eof or self.size > TEXT_LIMIT: + raise Damaged("compressed text") + + +def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: + """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" + if inflater.eof: + if data: + raise Damaged("data after the end of zlib stream") + return + out = inflater.decompress(data, INFLATE_CHUNK) + rows.feed(out) + while inflater.unconsumed_tail and not inflater.eof: + out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) + rows.feed(out) + if inflater.eof and inflater.unused_data: + raise Damaged("data after the end of zlib stream") diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..0f58f51 --- /dev/null +++ b/worker.py @@ -0,0 +1,527 @@ +"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose). + +Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1): +одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди. + +Обработка одного сообщения: + 1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ. + 2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый + lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op. + 3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает -> + аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued. + Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения. + 4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД. + 5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь). + 6. PUT в конечный prefix: attachments//v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp. + Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи. + 7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT. + 8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД. + 9. Удаление исходника из staging (если не вышло — доделает janitor). +Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД, +таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed. +""" +import hashlib +import json +import logging +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path + +import psycopg +from opentelemetry import trace +from opentelemetry.trace import NonRecordingSpan, SpanKind +from psycopg.rows import dict_row + +import settings +import telemetry + +telemetry.setup("messenger-worker") + +import broker # noqa: E402 +import faults # noqa: E402 +import imaging # noqa: E402 +import pipeline # noqa: E402 +import storage # noqa: E402 + +log = logging.getLogger("messenger.worker") + +# lab6: метрики фоновой обработки (contracts/observability.md). Метки: тип задания, исход, очередь. +JOB_ATTRS = {"job.type": pipeline.JOB_TYPE} +JOB_DURATION = telemetry.meter.create_histogram( + "messenger.job.duration", unit="s", + description="Attempt of an attachment job from claim to result (outcome: ready/retry/failed/stale/lease_lost)") +TIME_TO_READY = telemetry.meter.create_histogram( + "messenger.attachment.time_to_ready", unit="s", + description="From durable acceptance (job COMMIT in API) to ready COMMIT in worker") +MESSAGES = telemetry.meter.create_counter( + "messenger.messages.consumed", unit="{message}", + description="Broker deliveries by outcome: processed (claimed), noop (duplicate/not due), dead_lettered, requeued") +BUSY = {"value": 0} +BUSY_LOCK = threading.Lock() + + +def _observe_busy(_options): + from opentelemetry.metrics import Observation + return [Observation(BUSY["value"], {"messaging.destination.name": pipeline.QUEUE})] + + +def _observe_capacity(_options): + from opentelemetry.metrics import Observation + return [Observation(settings.WORKER_CONCURRENCY, {"messaging.destination.name": pipeline.QUEUE})] + + +telemetry.meter.create_observable_up_down_counter( + "messenger.worker.busy", [_observe_busy], unit="{consumer}", description="Consumers processing a job now") +telemetry.meter.create_observable_up_down_counter( + "messenger.worker.capacity", [_observe_capacity], unit="{consumer}", + description="Consumers of this worker process (WORKER_CONCURRENCY)") + + +def busy(delta: int) -> None: + with BUSY_LOCK: + BUSY["value"] += delta + +STOP = threading.Event() +HERE = Path(__file__).resolve().parent +VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} +IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large", + imaging.EXIT_MEMORY: "resource_limit"} + + +class JobError(Exception): + def __init__(self, code: str, permanent: bool): + super().__init__(code) + self.code = code + self.permanent = permanent + + +class LeaseLost(Exception): + pass + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def final_prefix(claim: pipeline.Claim) -> str: + return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/" + + +class LeaseKeeper(threading.Thread): + """Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку.""" + + def __init__(self, claim: pipeline.Claim): + super().__init__(daemon=True) + self.claim = claim + self.done = threading.Event() + self.lost = threading.Event() + self.frozen = threading.Event() # fault worker_freeze: перестать продлевать + + def run(self): + interval = max(1.0, settings.JOB_LEASE_SECONDS / 3) + conn = None + while not self.done.wait(interval): + if self.frozen.is_set(): + continue + try: + if conn is None: + conn = connect_db() + if not pipeline.renew_lease(conn, self.claim): + log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id, + extra={"event": "job.lease_lost", "job.id": str(self.claim.job_id)}) + self.lost.set() + return + except psycopg.Error as exc: + # Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT). + log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__) + if conn is not None: + conn.close() + conn = None + if conn is not None: + conn.close() + + def check(self): + if self.lost.is_set(): + raise LeaseLost() + + +def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: + # Дочернему процессу — только лимиты, никаких секретов окружения. + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1", + "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for fault in ("hang", "invalid", "oom"): + if faults.hit("imaging_" + fault): + env["FAULT_IMAGING"] = fault + with telemetry.internal_span("imaging", {"messenger.pipeline.version": claim.pipeline_version, + "messenger.source.content_type": claim.content_type}) as span: + result = _run_imaging(env, source, out_dir, claim, keeper) + span.set_attribute("messenger.variants", len(result["variants"])) + return result + + +def _run_imaging(env: dict, source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: + process = subprocess.Popen( + [sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type, + str(settings.MAX_IMAGE_PIXELS)], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir), + ) + deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS + try: + while True: + try: + output, _ = process.communicate(timeout=0.5) + break + except subprocess.TimeoutExpired: + if time.monotonic() > deadline: + raise JobError("processing_timeout", permanent=False) from None + keeper.check() + finally: + if process.poll() is None: + process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку + process.wait() + code = process.returncode + if code in IMAGING_EXIT: + raise JobError(IMAGING_EXIT[code], permanent=True) + if code != 0: + # Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены. + raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False) + return json.loads(output.decode("utf-8").strip().splitlines()[-1]) + + +def sha256_file(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with open(path, "rb") as file: + while chunk := file.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None: + if faults.hit("worker_s3_error"): + raise JobError("storage_unavailable", permanent=False) + with open(path, "rb") as file: + try: + storage.put(key, file, size, content_type, sha256) + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + + +def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]: + """Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов.""" + source = workdir / "source" + try: + storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES) + except storage.ObjectMissing: + raise JobError("source_missing", permanent=True) from None + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + with telemetry.internal_span("verify source sha256"): + size, digest = sha256_file(source) + if size != claim.size_bytes or digest != claim.sha256: + # Staging не совпадает с тем, что принял API: публиковать нельзя. + raise JobError("source_corrupted", permanent=True) + keeper.check() + + out_dir = workdir / "out" + out_dir.mkdir() + result = run_imaging(source, out_dir, claim, keeper) + keeper.check() + + prefix = final_prefix(claim) + original_key = prefix + "original" + put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes) + variants = [] + for item in result["variants"]: + path = out_dir / item["file"] + v_size, v_sha = sha256_file(path) + key = prefix + VARIANT_KEYS[item["name"]] + put_file(key, path, item["content_type"], v_sha, v_size) + variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size, + "width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key}) + keeper.check() + return original_key, variants + + +@dataclass +class Outcome: + ack: bool = True # False -> nack с requeue (не смогли даже записать claim) + reject: bool = False # True -> reject без requeue -> DLQ + cleanup_key: str | None = None + attachment_id: object = None + span_context: object = None # спан обработки: очистка staging после ack — в той же трассе + delivery: str = "processed" # метка messenger.messages.consumed: processed / noop + + +def carrier_from(headers, body: bytes) -> dict: + """W3C контекст сообщения: native headers AMQP (основной carrier), иначе поле traceparent тела.""" + carrier = {} + for key in ("traceparent", "tracestate"): + value = (headers or {}).get(key) + if isinstance(value, bytes): + value = value.decode("ascii", "replace") + if isinstance(value, str): + carrier[key] = value + if "traceparent" not in carrier: + try: + message = json.loads(body.decode("utf-8")) + if isinstance(message, dict) and isinstance(message.get("traceparent"), str): + carrier["traceparent"] = message["traceparent"] + except (UnicodeDecodeError, ValueError): + pass + return carrier + + +class Consumer(threading.Thread): + def __init__(self, number: int): + super().__init__(name=f"consumer-{number}", daemon=True) + self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}" + self.db = None + + def database(self): + if self.db is None or self.db.closed: + self.db = connect_db() + return self.db + + def handle(self, body: bytes, redelivered: bool, headers=None) -> Outcome: + """Спан CONSUMER «process attachments.process.v1» — ребёнок спана publish из dispatcher.""" + attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "process", + "messaging.destination.name": pipeline.QUEUE, "messaging.consumer.group.name": pipeline.QUEUE, + "messaging.rabbitmq.message.redelivered": redelivered} + with telemetry.tracer.start_as_current_span( + f"process {pipeline.QUEUE}", context=telemetry.extract(carrier_from(headers, body)), + kind=SpanKind.CONSUMER, attributes=attributes, + record_exception=False, set_status_on_exception=False) as span: + outcome = self._handle(body, redelivered, span) + outcome.span_context = span.get_span_context() + kind = "dead_lettered" if outcome.reject else "requeued" if not outcome.ack else outcome.delivery + span.set_attribute("messenger.delivery", kind) + MESSAGES.add(1, {"messaging.destination.name": pipeline.QUEUE, "outcome": kind}) + return outcome + + def _handle(self, body: bytes, redelivered: bool, span) -> Outcome: + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + log.warning("invalid message -> DLQ (%s)", exc, extra={"event": "job.invalid_message"}) + telemetry.mark_error(span, "invalid_message") + return Outcome(reject=True) + span.set_attributes({"messaging.message.id": f"{message['job_id']}:{message['attempt']}", + "messenger.job.id": message["job_id"], + "messenger.attachment.id": message["attachment_id"], + "messenger.message.attempt": message["attempt"]}) + try: + result = pipeline.claim(self.database(), message, self.owner, redelivered) + except psycopg.Error as exc: + log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__, + extra={"event": "job.claim_failed", "job.id": message["job_id"], + "error.type": type(exc).__name__}) + telemetry.mark_error(span, type(exc).__name__) + self.db = None + STOP.wait(2) + return Outcome(ack=False) + if isinstance(result, pipeline.Reject): + log.warning("job %s: %s -> DLQ", message["job_id"], result.reason, + extra={"event": "job.rejected", "job.id": message["job_id"], "reason": result.reason}) + telemetry.mark_error(span, result.reason) + return Outcome(reject=True) + if isinstance(result, pipeline.Skip): + log.info("job %s: no-op (%s), ack", message["job_id"], result.reason, + extra={"event": "job.noop", "job.id": message["job_id"], "reason": result.reason}) + span.set_attribute("messenger.noop.reason", result.reason) + return Outcome(delivery="noop") + span.set_attribute("messenger.job.attempt", result.attempt) + busy(+1) + try: + return self.execute(result, span) + finally: + busy(-1) + + def execute(self, claim: pipeline.Claim, span) -> Outcome: + started = time.monotonic() + fields = {"job.id": str(claim.job_id), "attachment.id": str(claim.attachment_id), "attempt": claim.attempt} + log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt, + claim.max_attempts, claim.attachment_id, extra={"event": "job.claimed", **fields}) + keeper = LeaseKeeper(claim) + keeper.start() + workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR)) + outcome_name = "ready" + try: + try: + original_key, variants = process_job(claim, keeper, workdir) + faults.crash("worker_crash_after_put") + if faults.hit("worker_freeze"): + keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят + time.sleep(settings.JOB_LEASE_SECONDS * 3) + with telemetry.internal_span("commit result"): + time_to_ready = pipeline.finish_ready(self.database(), claim, original_key, variants, + self.owner) + if time_to_ready is None: + outcome_name = "stale" # аренду перехватили: fencing отклонил результат + except JobError as exc: + outcome_name = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner) + telemetry.mark_error(span, exc.code) + log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code, + "permanent" if exc.permanent else "temporary", outcome_name, + extra={"event": f"job.{outcome_name}", "error.type": exc.code, + "permanent": exc.permanent, **fields}) + return Outcome() + except LeaseLost: + outcome_name = "lease_lost" + telemetry.mark_error(span, "lease_lost") + log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt, + extra={"event": "job.lease_lost", **fields}) + return Outcome() + except psycopg.Error: + raise + except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены + log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt, + extra={"event": "job.unexpected_error", **fields}) + telemetry.mark_error(span, "processing_failed") + outcome_name = pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner) + return Outcome() + except psycopg.Error as exc: + # Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и + # dispatcher вернёт job в очередь; если прошёл — повтор увидит ready. + outcome_name = "db_error" + telemetry.mark_error(span, type(exc).__name__) + log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__, + extra={"event": "job.db_error", "error.type": type(exc).__name__, **fields}) + self.db = None + return Outcome() + finally: + keeper.done.set() + shutil.rmtree(workdir, ignore_errors=True) + elapsed = time.monotonic() - started + span.set_attribute("messenger.job.outcome", outcome_name) + JOB_DURATION.record(elapsed, JOB_ATTRS | {"outcome": outcome_name}) + if time_to_ready is None: + telemetry.mark_error(span, "stale_result_rejected") + log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id, + claim.attempt, extra={"event": "job.stale_result_rejected", **fields}) + return Outcome() + TIME_TO_READY.record(time_to_ready, JOB_ATTRS) + faults.crash("worker_crash_after_commit") + log.info("job %s attempt %s: ready in %.0f ms (%.1f s since upload)", claim.job_id, claim.attempt, + (time.monotonic() - started) * 1000, time_to_ready, + extra={"event": "job.ready", "duration_ms": round((time.monotonic() - started) * 1000, 1), + "time_to_ready_s": round(time_to_ready, 3), **fields}) + return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id) + + def cleanup_staging(self, outcome: Outcome) -> None: + """После COMMIT ready и ack: исходник в staging больше не нужен.""" + parent = trace.set_span_in_context(NonRecordingSpan(outcome.span_context)) if outcome.span_context else None + with telemetry.tracer.start_as_current_span("cleanup staging", context=parent): + try: + storage.delete(outcome.cleanup_key) + self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' " + "AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key)) + except (storage.StorageUnavailable, psycopg.Error) as exc: + log.info("staging cleanup postponed to janitor: %s", type(exc).__name__, + extra={"event": "staging.cleanup_postponed", "error.type": type(exc).__name__}) + self.db = None + + def consume_once(self) -> None: + connection = broker.connect() + try: + channel = connection.channel() + channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH) + broker.check_topology(channel) + log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH, + extra={"event": "worker.consuming"}) + for method, properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1): + if STOP.is_set(): + break + if method is None: + continue + box = {} + headers = properties.headers if properties is not None else None + runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle( + body, method.redelivered, headers)), daemon=True) + runner.start() + while runner.is_alive(): + connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа + outcome = box.get("outcome", Outcome(ack=False)) + if outcome.reject: + channel.basic_reject(method.delivery_tag, requeue=False) + elif outcome.ack: + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + if outcome.cleanup_key: + self.cleanup_staging(outcome) + try: + channel.cancel() + except broker.BrokerError: + pass + finally: + try: + connection.close() + except broker.BrokerError: + pass + + def safe_handle(self, body, redelivered, headers=None) -> Outcome: + try: + return self.handle(body, redelivered, headers) + except Exception: # noqa: BLE001 + log.exception("unexpected error while handling a message; requeue", + extra={"event": "job.handler_crashed"}) + self.db = None + STOP.wait(2) + return Outcome(ack=False) + + def run(self): + pause = 1.0 + while not STOP.is_set(): + try: + self.consume_once() + pause = 1.0 + except broker.BrokerError as exc: + log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name, + type(exc).__name__, pause, + extra={"event": "broker.unavailable", "error.type": type(exc).__name__}) + STOP.wait(pause) + pause = min(pause * 2, 15) + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True) + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)] + for consumer in consumers: + consumer.start() + log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, " + "fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH, + settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB, + "ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"}) + while not STOP.wait(2): + if not all(consumer.is_alive() for consumer in consumers): + log.error("a consumer thread died, exiting so that Docker restarts the worker") + os._exit(1) + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят. + for consumer in consumers: + consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10) + log.info("stopped", extra={"event": "service.stopped"}) + telemetry.shutdown() + + +if __name__ == "__main__": + main() -- 2.54.0