From 9cc80d2529c73044f06afef3bd43ef75d6dcf3d1 Mon Sep 17 00:00:00 2001 From: yura Date: Wed, 18 Nov 2026 23:59:35 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=207=20=E2=80=94=20=D0=91=D0=9F?= =?UTF-8?q?=D0=982303=20=E2=80=94=20=D0=9B=D0=B0=D0=B7=D0=B0=D1=80=D0=B5?= =?UTF-8?q?=D0=B2=20=E2=80=94=20=D0=BD=D0=B0=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS. - Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2. - Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1. - TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert, mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape. - OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором, auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login. - Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и пользователи брокера выдаются из OpenBao. - Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе. - Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL, no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API. - Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao, hardening; evidence, в т.ч. прогон из чистого clone. --- .env.example | 59 +- .github/pull_request_template.md | 28 +- .github/workflows/app.yml | 48 + Dockerfile | 40 + REPORT.md | 186 +- compose.yaml | 842 +++++ contracts/trust-matrix.md | 60 +- evidence/README.md | 21 + evidence/argon2-bench.json | 120 + evidence/check-rules.txt | 19 + evidence/clean-clone-run.txt | 88 + evidence/dashboard-check.json | 598 +++ evidence/dashboard-check.txt | 84 + evidence/lab7-api-security.json | 173 + evidence/lab7-api-security.txt | 34 + evidence/lab7-hardening.json | 917 +++++ evidence/lab7-hardening.txt | 183 + evidence/lab7-openbao.json | 422 +++ evidence/lab7-openbao.txt | 84 + evidence/lab7-tls-matrix.json | 607 +++ evidence/lab7-tls-matrix.txt | 68 + evidence/load-after-nodelay.json | 17 + evidence/load-before-nodelay.json | 17 + evidence/load.json | 17 + evidence/load.txt | 17 + evidence/make-check.txt | 2 + evidence/make-test-security.txt | 2 + evidence/make-test.txt | 2 + evidence/obs-correlation.json | 238 ++ evidence/obs-correlation.trees.txt | 33 + evidence/obs-correlation.txt | 8 + evidence/persistence-ps-after.txt | 16 + evidence/persistence-ps-before.txt | 16 + evidence/prometheus-up.txt | 7 + evidence/run-all.txt | 61 + evidence/scan-secrets.txt | 1 + evidence/test-failover.txt | 4 + evidence/test-persistence.txt | 3 + evidence/worker-outage.txt | 11 + infra/grafana/dashboards/messenger.json | 3299 +++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 15 + .../provisioning/datasources/datasources.yaml | 103 + infra/haproxy/haproxy.cfg | 122 + infra/loki/loki.yaml | 81 + infra/openbao/README.md | 62 +- infra/openbao/openbao.hcl | 36 + infra/openbao/policies/messenger-api.hcl | 5 + infra/openbao/policies/messenger-janitor.hcl | 5 + infra/openbao/policies/messenger-migrate.hcl | 5 + infra/openbao/policies/messenger-ops.hcl | 5 + infra/openbao/policies/messenger-relay.hcl | 5 + infra/openbao/policies/messenger-s3-init.hcl | 5 + infra/openbao/policies/messenger-worker.hcl | 5 + infra/otel/README.md | 15 + infra/otel/collector.yaml | 169 + infra/postgres/pg_hba.conf | 12 + infra/postgres/start.sh | 17 + infra/prometheus/client.yml | 6 + infra/prometheus/prometheus.yml | 97 + infra/prometheus/rules/messenger.yml | 158 + infra/prometheus/tests/messenger.test.yml | 82 + infra/prometheus/web.yml | 8 + infra/rabbitmq/rabbitmq.conf | 30 + infra/runbooks/README.md | 128 + infra/tempo/tempo.yaml | 55 + migrations/001_init.sql | 40 + migrations/002_idempotency.sql | 17 + migrations/003_attachments.sql | 40 + migrations/004_async_processing.sql | 126 + migrations/security/grants.sql | 48 + requirements.txt | 22 + scripts/bao-init.sh | 77 + scripts/bao-unseal.sh | 25 + scripts/build_dashboard.py | 323 ++ scripts/check_rules.sh | 30 + scripts/init-secrets.sh | 135 + scripts/pki-issue.sh | 94 + scripts/restart.sh | 14 + scripts/scan_secrets.py | 64 + scripts/start-all.sh | 7 + scripts/start-worker.sh | 6 + scripts/stop-one.sh | 25 + scripts/stop-worker.sh | 7 + security/THREAT_MODEL.md | 65 +- src/common.py | 107 + src/images.py | 282 ++ src/instance.py | 20 + src/janitor.py | 242 ++ src/jobs.py | 451 +++ src/jpeg_probe.py | 52 + src/migrate.py | 66 + src/objstore.py | 233 ++ src/ops.py | 220 ++ src/relay.py | 347 ++ src/s3_bootstrap.py | 87 + src/security.py | 81 + src/selfcheck.py | 130 + src/server.py | 1401 +++++++ src/store.py | 470 +++ src/telemetry.py | 492 +++ src/tlsutil.py | 55 + src/transform.py | 128 + src/vault.py | 195 + src/worker.py | 622 ++++ student-tests/argon2_bench.py | 89 + student-tests/broker_permissions.py | 99 + student-tests/burst.py | 124 + student-tests/cursor_validation.py | 32 + student-tests/dashboard_check.py | 103 + student-tests/drain_bench.py | 69 + student-tests/fixtures/sample.jpg | Bin 0 -> 8265 bytes student-tests/incident_db_slow.py | 208 ++ student-tests/incident_worker_stop.py | 164 + student-tests/janitor_replay_race.py | 120 + student-tests/lab5lib.py | 175 + student-tests/lab7_api_security.py | 256 ++ student-tests/lab7_hardening.py | 186 + student-tests/lab7_openbao.py | 148 + student-tests/lab7_tls_matrix.py | 273 ++ student-tests/lease_takeover.py | 80 + student-tests/legacy/README.md | 18 + student-tests/legacy/api_negative_lab5.py | 161 + student-tests/legacy/attachment_failover.py | 72 + student-tests/legacy/attachments_negative.py | 217 ++ student-tests/legacy/failover-each.sh | 50 + student-tests/legacy/failover_bench.py | 264 ++ .../legacy/idempotency_concurrency.py | 130 + student-tests/legacy/login_flood.py | 172 + student-tests/legacy/progressive_upload.py | 77 + student-tests/legacy/queue_contract_check.py | 98 + student-tests/legacy/s3_outage.py | 106 + student-tests/legacy/s3_private_check.py | 114 + student-tests/legacy/slow_client.py | 102 + student-tests/legacy/upload_memory.py | 204 + student-tests/legacy/variants_check.py | 187 + student-tests/obs_correlation.py | 206 + student-tests/obslib.py | 160 + student-tests/outbox_gap.py | 161 + student-tests/run_all.sh | 40 + student-tests/s3_tamper.py | 36 + student-tests/telemetry_outage.py | 192 + student-tests/telemetry_persistence.py | 92 + student-tests/traffic.py | 140 + student-tests/transform_limits.py | 101 + student-tests/worker_faults.py | 172 + student-tests/worker_outage.py | 93 + 146 files changed, 21935 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/app.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/argon2-bench.json create mode 100644 evidence/check-rules.txt create mode 100644 evidence/clean-clone-run.txt create mode 100644 evidence/dashboard-check.json create mode 100644 evidence/dashboard-check.txt create mode 100644 evidence/lab7-api-security.json create mode 100644 evidence/lab7-api-security.txt create mode 100644 evidence/lab7-hardening.json create mode 100644 evidence/lab7-hardening.txt create mode 100644 evidence/lab7-openbao.json create mode 100644 evidence/lab7-openbao.txt create mode 100644 evidence/lab7-tls-matrix.json create mode 100644 evidence/lab7-tls-matrix.txt create mode 100644 evidence/load-after-nodelay.json create mode 100644 evidence/load-before-nodelay.json create mode 100644 evidence/load.json create mode 100644 evidence/load.txt create mode 100644 evidence/make-check.txt create mode 100644 evidence/make-test-security.txt create mode 100644 evidence/make-test.txt create mode 100644 evidence/obs-correlation.json create mode 100644 evidence/obs-correlation.trees.txt create mode 100644 evidence/obs-correlation.txt create mode 100644 evidence/persistence-ps-after.txt create mode 100644 evidence/persistence-ps-before.txt create mode 100644 evidence/prometheus-up.txt create mode 100644 evidence/run-all.txt create mode 100644 evidence/scan-secrets.txt create mode 100644 evidence/test-failover.txt create mode 100644 evidence/test-persistence.txt create mode 100644 evidence/worker-outage.txt create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/loki/loki.yaml create mode 100644 infra/openbao/openbao.hcl create mode 100644 infra/openbao/policies/messenger-api.hcl create mode 100644 infra/openbao/policies/messenger-janitor.hcl create mode 100644 infra/openbao/policies/messenger-migrate.hcl create mode 100644 infra/openbao/policies/messenger-ops.hcl create mode 100644 infra/openbao/policies/messenger-relay.hcl create mode 100644 infra/openbao/policies/messenger-s3-init.hcl create mode 100644 infra/openbao/policies/messenger-worker.hcl create mode 100644 infra/otel/collector.yaml create mode 100644 infra/postgres/pg_hba.conf create mode 100755 infra/postgres/start.sh create mode 100644 infra/prometheus/client.yml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules/messenger.yml create mode 100644 infra/prometheus/tests/messenger.test.yml create mode 100644 infra/prometheus/web.yml create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 infra/runbooks/README.md create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_async_processing.sql create mode 100644 migrations/security/grants.sql create mode 100644 requirements.txt create mode 100755 scripts/bao-init.sh create mode 100755 scripts/bao-unseal.sh create mode 100755 scripts/build_dashboard.py create mode 100755 scripts/check_rules.sh create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/pki-issue.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/scan_secrets.py create mode 100755 scripts/start-all.sh create mode 100755 scripts/start-worker.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/stop-worker.sh create mode 100644 src/common.py create mode 100644 src/images.py create mode 100644 src/instance.py create mode 100644 src/janitor.py create mode 100644 src/jobs.py create mode 100644 src/jpeg_probe.py create mode 100644 src/migrate.py create mode 100644 src/objstore.py create mode 100644 src/ops.py create mode 100644 src/relay.py create mode 100644 src/s3_bootstrap.py create mode 100644 src/security.py create mode 100644 src/selfcheck.py create mode 100644 src/server.py create mode 100644 src/store.py create mode 100644 src/telemetry.py create mode 100644 src/tlsutil.py create mode 100644 src/transform.py create mode 100644 src/vault.py create mode 100644 src/worker.py create mode 100644 student-tests/argon2_bench.py create mode 100644 student-tests/broker_permissions.py create mode 100644 student-tests/burst.py create mode 100644 student-tests/cursor_validation.py create mode 100644 student-tests/dashboard_check.py create mode 100644 student-tests/drain_bench.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/incident_db_slow.py create mode 100644 student-tests/incident_worker_stop.py create mode 100644 student-tests/janitor_replay_race.py create mode 100644 student-tests/lab5lib.py create mode 100644 student-tests/lab7_api_security.py create mode 100644 student-tests/lab7_hardening.py create mode 100644 student-tests/lab7_openbao.py create mode 100644 student-tests/lab7_tls_matrix.py create mode 100644 student-tests/lease_takeover.py create mode 100644 student-tests/legacy/README.md create mode 100644 student-tests/legacy/api_negative_lab5.py create mode 100644 student-tests/legacy/attachment_failover.py create mode 100644 student-tests/legacy/attachments_negative.py create mode 100755 student-tests/legacy/failover-each.sh create mode 100644 student-tests/legacy/failover_bench.py create mode 100644 student-tests/legacy/idempotency_concurrency.py create mode 100644 student-tests/legacy/login_flood.py create mode 100644 student-tests/legacy/progressive_upload.py create mode 100644 student-tests/legacy/queue_contract_check.py create mode 100644 student-tests/legacy/s3_outage.py create mode 100644 student-tests/legacy/s3_private_check.py create mode 100644 student-tests/legacy/slow_client.py create mode 100644 student-tests/legacy/upload_memory.py create mode 100644 student-tests/legacy/variants_check.py create mode 100644 student-tests/obs_correlation.py create mode 100644 student-tests/obslib.py create mode 100644 student-tests/outbox_gap.py create mode 100755 student-tests/run_all.sh create mode 100644 student-tests/s3_tamper.py create mode 100644 student-tests/telemetry_outage.py create mode 100644 student-tests/telemetry_persistence.py create mode 100644 student-tests/traffic.py create mode 100644 student-tests/transform_limits.py create mode 100644 student-tests/worker_faults.py create mode 100644 student-tests/worker_outage.py diff --git a/.env.example b/.env.example index 2e9f054..db1bc66 100644 --- a/.env.example +++ b/.env.example @@ -1,41 +1,38 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 +# Lab7: копируется в .env (scripts/init-secrets.sh) для ЛОКАЛЬНОГО синтетического стенда; .env не коммитится. +# Здесь НЕТ секретов: пароли/ключи — в secrets/ (вне Git) и в OpenBao; CA keys и unseal shares — +# в каталоге оператора ($OPERATOR_DIR, по умолчанию ~/.mtusi2026lab7-operator). +# Соответствие логическим именам задания — REPORT.md «Архитектура и решения». APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +APP_VERSION=1 API_REPLICAS=2 +WORKER_REPLICAS=2 +# Публичный HTTPS ingress (HAProxy, сертификат public-ca) и Grafana (HTTPS): только 127.0.0.1. +LB_PORT=8443 +GRAFANA_PORT=3000 +SESSION_TTL_SECONDS=3600 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=https://s3:9000 -S3_REGION=us-east-1 +# Ограничение входа на LB: попыток POST /auth/sessions за 10 с с одного IP (сверх — 429 + Retry-After). +LOGIN_RATE_LIMIT=20 +# Argon2id (не ниже базового профиля m=19 MiB, t=2, p=1 — процесс иначе не стартует). +ARGON2_MEMORY_KIB=19456 +ARGON2_ITERATIONS=2 +ARGON2_PARALLELISM=1 +PASSWORD_HASH_CONCURRENCY=2 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_REGION=us-east-1 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab -# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix. -PUBLIC_BASE_URL=https://localhost:8443 -TLS_CA_FILE=/run/secrets/ca.crt -TLS_CERT_FILE=/run/secrets/service.crt -TLS_KEY_FILE=/run/secrets/service.key -BAO_ADDR=https://openbao:8200 -BAO_CACERT=/run/secrets/bao-ca.crt -# No root token here. Bootstrap operator -> short-lived service auth -> scoped token. -ARGON2_MEMORY_KIB=19456 -ARGON2_ITERATIONS=2 -ARGON2_PARALLELISM=1 +OTEL_SDK_DISABLED=false +# Сколько процесс ждёт unsealed OpenBao на старте, прежде чем завершиться (не берёт значений по умолчанию). +BAO_STARTUP_WAIT_SECONDS=300 +# Фиксированные в compose.yaml (не переопределяются .env), для справки: +# OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318 (+ CA и client cert сервиса) +# S3_ENDPOINT=https://s3:8333, POSTGRES_HOST=db (sslmode=verify-full), BAO_ADDR=https://openbao:8200 +# TLS_CA_FILE=/run/secrets/internal-ca.crt, TLS_CERT_FILE=/run/secrets/service.crt, TLS_KEY_FILE=/run/secrets/service.key +# Учебные инъекции отказов (по умолчанию выключены): +FAULT_INJECTION=0 +WORKER_FAULT_INJECTION=0 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8aef0ff..76263f7 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 7 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Лазарев Юрий Сергеевич, БПИ2303 +Целевая оценка (3/4/5): 3 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесена реализация lab6 (API ×2 за HAProxy, PostgreSQL, SeaweedFS S3, RabbitMQ, relay outbox, worker ×2, janitor, OTel Collector, Prometheus, Loki, Tempo, Grafana) без изменения бизнес-API. Добавлено: внешний HTTPS (public-ca) на 127.0.0.1:8443; обязательное mTLS LB→каждая API-реплика с allowlist identity `lb` (проверка SAN в приложении); TLS с проверкой сервера ко всем зависимостям (PostgreSQL verify-full + pg_hba только hostssl, HTTPS S3, AMQPS, OTLP/HTTPS с client cert, mTLS Collector→backends, Grafana→datasources, Prometheus scrape); OpenBao 2.7.2 не в dev-mode (Raft на volume, ручной init/unseal оператором, auth/cert с ролью и policy на сервис, TTL 15 мин + renew); раздельные роли PostgreSQL с грантами по таблицам; Argon2id (m=19 MiB, t=2, p=1, замер на стенде) с rehash; случайный X-Instance-Id; hardening (non-root, cap_drop ALL, no-new-privileges, read-only rootfs, tmpfs, лимиты, только 2 порта на 127.0.0.1); модель угроз и trust matrix; отрицательные тесты. +Команды запуска из чистого clone: `./scripts/init-secrets.sh && ./scripts/bao-init.sh && docker compose --env-file .env up -d --build --wait && make check test test-security CA_FILE=$PWD/secrets/pki/public-ca.crt` — подробно в REPORT.md «Запуск из чистого clone». +Результаты проверок и ссылка на REPORT.md: REPORT.md «Доказательства критериев»; stdout — evidence/*.txt, агрегаты — evidence/*.json (полный прогон `./student-tests/run_all.sh`). +Ограничения / известные ошибки: REPORT.md «Ограничения и следующие шаги» (Collector/backends/RabbitMQ проверяют только CA, не роль клиента; нет ротации/CRL; unseal shares у оператора на том же хосте; часть нагрузочных тестов lab4–5 не перенесена на TLS — student-tests/legacy). +Источники и использование ИИ: REPORT.md, шапка; код подготовлен с помощью ИИ-ассистента (Claude), запуск и проверки — автор. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 0000000..c558159 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,48 @@ +# Проверка РЕАЛИЗАЦИИ lab7 (contracts.yml проверяет только skeleton). Изолированный стенд Compose +# с одноразовыми секретами и PKI, созданными на раннере; OPERATOR_DIR — временный каталог раннера +# (роль «оператора» в CI играет сам job). Секреты CI не нужны. pull_request_target не используется. +# Ограничение: локально workflow не исполнялся (нет раннера); проверен синтаксис YAML. +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + app: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + COMPOSE_PROJECT_NAME: mtusi-lab7-ci + OPERATOR_DIR: ${{ runner.temp }}/operator + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Secrets, PKI, config + run: ./scripts/init-secrets.sh && docker compose --env-file .env config --quiet + - name: Telemetry configs and alert rule unit tests + run: ./scripts/check_rules.sh + - name: Build + run: docker compose build migrate + - name: OpenBao init/unseal/bootstrap (operator step) + run: ./scripts/bao-init.sh + - name: Start + run: docker compose --env-file .env up -d --wait --wait-timeout 300 + - name: Public HTTPS smoke and external TLS trust + run: make check test test-security CA_FILE=$PWD/secrets/pki/public-ca.crt + - name: Internal TLS/mTLS matrix + run: docker compose --profile tools run --rm -T checks /app/student-tests/lab7_tls_matrix.py + - name: API negative tests + run: python3 student-tests/lab7_api_security.py --ca-file secrets/pki/public-ca.crt + - name: OpenBao policies and sealed behaviour + run: python3 student-tests/lab7_openbao.py --ca-file secrets/pki/public-ca.crt + - name: Hardening and secret leaks + run: CA_FILE=$PWD/secrets/pki/public-ca.crt python3 student-tests/lab7_hardening.py + - name: Persistence (HTTPS) + run: make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE=$PWD/secrets/pki/public-ca.crt + - name: Logs on failure (application services only; secrets are never logged) + if: failure() + run: docker compose logs --no-color --tail=200 api worker relay migrate s3-init lb openbao + - name: Cleanup + if: always() + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fd97aa5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# Lab7: мессенджер на Python + PostgreSQL + S3 + RabbitMQ + OpenTelemetry SDK + OpenBao (секреты) + mTLS. Один образ для API-реплик, relay, +# worker, janitor, ops-CLI и job-ов migrate (схема), s3-init (bucket). +FROM python:3.12.7-slim@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d + +# MALLOC_ARENA_MAX=2: glibc не заводит арену на каждый поток ThreadingHTTPServer — +# освобождённые буферы upload возвращаются в общий пул, RSS не растёт с числом потоков. +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + MALLOC_ARENA_MAX=2 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8443 + +WORKDIR /app + +# Сначала зависимости (закреплённые версии) — слой кэшируется, пока не изменится requirements.txt. +COPY requirements.txt /app/ +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ /app/src/ +COPY migrations/ /app/migrations/ +# Временные файлы upload (O_TMPFILE) и рабочие файлы worker — в /tmp. Lab7: rootfs контейнеров +# read-only, /tmp — tmpfs (compose.yaml); каталоги создаёт сам процесс при старте. +ENV UPLOAD_TMP_DIR=/tmp/uploads + +# Метка версии сборки (rolling update: scripts/rolling-update.sh собирает с новым значением). +ARG APP_VERSION=1 +ENV APP_VERSION=${APP_VERSION} + +# Lab7: в образе нет секретов, ключей и сертификатов (.dockerignore исключает secrets/, *.key, *.pem); +# всё это приходит в runtime: сертификат/ключ — Docker secret, пароли/ключи — из OpenBao. +RUN useradd --no-create-home --uid 10001 app +USER 10001:10001 + +# 8443 — mTLS (только identity lb); 8081 — health только на 127.0.0.1 контейнера. +EXPOSE 8443 +# Python-процесс — PID 1 (exec-форма), сам обрабатывает SIGTERM (graceful shutdown). +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=2s --start-period=10s --retries=2 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"] +CMD ["python", "/app/src/server.py"] diff --git a/REPORT.md b/REPORT.md index 9801db4..7e39ce6 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,195 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Лазарев Юрий Сергеевич, БПИ2303 +- Целевая оценка; выбранный трек 5, если нужен: **3**, трек 5 не заявлен (отдельные пункты уровня 4 сделаны попутно и отмечены, уровень 4 целиком не заявляется) +- Commit/PR: TODO после публикации (fork `zemiles/mtusi2026lab7`, ветка `solution/bpi2303-lazarev`) +- Использованные источники, библиотеки и помощь ИИ: всё из lab6 (Python 3.12, psycopg 3.2.3, psycopg-pool 3.2.4, minio 7.2.20, urllib3 2.8.0, Pillow 13.0.0, pika 1.3.2, OpenTelemetry SDK 1.45.1) и **argon2-cffi 25.1.0** (+ argon2-cffi-bindings 25.1.0). Новый образ: `openbao/openbao:2.7.2@sha256:50469110…0561f`; остальные образы и digest — как в lab6 (см. `compose.yaml`). Документация: OWASP Password Storage / File Upload Cheat Sheets, OpenBao (auth/cert, policies, KV v2, integrated storage, production hardening, declarative audit), PostgreSQL (SSL, pg_hba, привилегии, `SELECT … FOR UPDATE`), RabbitMQ TLS и rabbitmq_prometheus TLS, HAProxy SSL (`verify`, `verifyhost`, `crt`), OTel Collector `configtls`, Prometheus web-config, Loki/Tempo `http_tls_config`, Grafana provisioning `$__file{}`, Docker security (capabilities, no-new-privileges, read-only). Код lab6 перенесён и доработан с помощью ИИ-ассистента (Claude); решения, запуск и проверки — автор. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверено: MacBook Air M3, 16 GB RAM, macOS 26.4.1; Docker Desktop 28.0.4 (VM 8 vCPU, 7.7 GiB), Compose v2.34.0; Python 3.12 на хосте (только стандартная библиотека, для тестов). Порты `127.0.0.1:8443` (HTTPS API) и `127.0.0.1:3000` (HTTPS Grafana) должны быть свободны — стенд lab6 остановить. + +```sh +# 0. Локальные секреты, PKI и .env (не коммитятся; значения не печатаются; повтор ничего не перезаписывает). +# CA private keys -> $OPERATOR_DIR (по умолчанию ~/.mtusi2026lab7-operator, ВНЕ проекта). +# Сертификаты выпускает OpenSSL закреплённого образа python:3.12.7-slim (docker run --network none). +./scripts/init-secrets.sh + +# 1. Офлайн-проверка конфигов (promtool config/web-config/rules, otelcol validate, loki -verify-config). +./scripts/check_rules.sh + +# 2. ОПЕРАТОР, один раз на пустом volume: OpenBao init (3 shares, порог 2) -> $OPERATOR_DIR/openbao/init.json, +# unseal, KV v2, policies, роли auth/cert, запись сервисных секретов. Повтор безопасен. +docker compose build migrate # образ приложения (собирает его только migrate) +./scripts/bao-init.sh + +# 3. Запуск (~40 с до готовности 18 контейнеров) +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +# порядок: openbao (unsealed) + db, s3, broker (healthy) -> migrate (роли+гранты), s3-init -> api x2, +# relay, worker x2, janitor -> lb; телеметрия независимо: otel-collector, prometheus, loki, tempo -> grafana +docker compose ps # migrate и s3-init Exited (0); порты только у lb и grafana + +# 4. Обязательные проверки (публичный CA — secrets/pki/public-ca.crt) +CA=$PWD/secrets/pki/public-ca.crt +make check && make test CA_FILE=$CA && make test-security CA_FILE=$CA +make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE=$CA # рестарт ВСЕХ хранилищ, вкл. OpenBao (+unseal) +make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=$CA && ./scripts/start-all.sh + +# 5. Проверки lab7 +docker compose --profile tools run --rm -T checks /app/student-tests/lab7_tls_matrix.py --output /app/evidence/lab7-tls-matrix.json +python3 student-tests/lab7_api_security.py --ca-file $CA --output evidence/lab7-api-security.json +python3 student-tests/lab7_openbao.py --ca-file $CA --output evidence/lab7-openbao.json +CA_FILE=$CA python3 student-tests/lab7_hardening.py --output evidence/lab7-hardening.json +docker compose exec -T api python - < student-tests/argon2_bench.py > evidence/argon2-bench.json + +# Всё сразу с записью stdout в evidence/*.txt: +./student-tests/run_all.sh # обязательное + lab7 (≈ 8 мин) +WITH_LAB6=1 ./student-tests/run_all.sh # + перенесённые проверки lab6/lab5 поверх TLS + +# 6. Grafana: https://localhost:3000 (доверить secrets/pki/public-ca.crt), admin / secrets/grafana_admin_password. + +# 7. После рестарта Docker/хоста: OpenBao sealed -> ОПЕРАТОР +./scripts/bao-unseal.sh + +# 8. Безопасная остановка (данные и секреты OpenBao остаются в volumes) +docker compose stop # или docker compose down (volumes остаются) + +# 9. Намеренное удаление учебных данных (НЕОБРАТИМО): volumes, затем материал оператора +docker compose down -v # после этого init.json в $OPERATOR_DIR/openbao не соответствует стенду — + # удалить его вручную перед новым bao-init.sh +``` + +**Действия оператора.** Только оператор имеет `$OPERATOR_DIR` (ключи CA, root token, unseal shares). Нужны: `bao-init.sh` один раз на новый volume, `bao-unseal.sh` после каждого рестарта контейнера OpenBao (Docker/хост). Обычный рестарт API/worker ничего не требует. `restart.sh` — операторский хук, поэтому сам делает unseal. + +**Замечание (Docker Desktop на macOS)**: как в lab6, при запуске из неинтерактивной среды `docker pull` зависал на keychain-хелпере; обход — отдельный `DOCKER_CONFIG` с `config.json = {}` и ссылкой `cli-plugins`. Публичные образы скачиваются анонимно. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` + хост 127.0.0.1:8443 ─HTTPS(public-ca)─► lb ═mTLS: lb предъявляет id "lb", проверяет SAN "api"═► api ×2 + │ сеть apinet (internal: только lb и api) │ + хост 127.0.0.1:3000 ─HTTPS(public-ca)─► grafana ═mTLS═► prometheus / loki / tempo │ + │ + api/relay/worker/janitor ──TLS verify-full + роль *_svc──► db (PostgreSQL, hostssl only) ◄──────┤ + api/worker/janitor ──HTTPS + свой ключ──► s3 (SeaweedFS) │ + relay/worker ──AMQPS 5671 + свой пользователь/ACL──► broker (RabbitMQ) │ + все процессы приложения ═OTLP/HTTPS + client cert═► otel-collector ═mTLS═► tempo/loki/prometheus │ + api/relay/worker/janitor/migrate/s3-init/ops ──TLS + auth/cert──► openbao (сеть vault) ◄─────────┘ +``` + +Источник истины, транзакционные границы, outbox, at-least-once и наблюдаемость — без изменений с lab5/lab6. Изменения lab7: + +| Область | Решение | Где | +| --- | --- | --- | +| PKI | Два CA: **public-ca** (ingress, Grafana; его сертификат раздаётся клиентам) и **internal-ca** (сервисы). ECDSA P-256, leaf 30 дней, CA 365 дней, SAN DNS = имя сервиса = identity. Ключи CA — только у оператора. Тестовые identities: `intruder` (internal-ca, чужая роль), `rogue-lb` (чужой CA, SAN `lb`) | `scripts/init-secrets.sh`, `scripts/pki-issue.sh` | +| Внешний TLS | HAProxy `:8443 ssl`, TLS ≥ 1.2, без tickets, HSTS, без `Server` | `infra/haproxy/haproxy.cfg` | +| LB → API | mTLS обязательный: LB проверяет CA + `verifyhost api`, предъявляет `lb.pem`; API — `CERT_REQUIRED` и **SAN ∈ MTLS_ALLOWED_CLIENTS (= lb)**, иначе закрытие до HTTP. Handshake в потоке соединения, таймаут 5 с. Health — отдельный plaintext listener **только 127.0.0.1:8081** | `src/server.py` (`Server.finish_request`, `HealthHandler`), `src/tlsutil.py` | +| Секреты | OpenBao 2.7.2: Raft на volume, TLS, audit → stdout; **auth/cert** (identity = leaf сервиса), policy на сервис = read только своего `kv/data/messenger/`; токен 15 мин/max 1 ч, renew, re-login. В env/образе/Docker secrets приложений паролей нет | `infra/openbao/`, `scripts/bao-init.sh`, `src/vault.py` | +| PostgreSQL | `ssl=on`, pg_hba: `hostssl … scram-sha-256`, `hostnossl reject`, локально `peer` только для `postgres`. Роли `api_svc/worker_svc/relay_svc/janitor_svc/ops_svc` + гранты по таблицам; для `SELECT … FOR UPDATE` — только `UPDATE(id)`. DDL — job `migrate` (роль `postgres`) | `infra/postgres/`, `migrations/security/grants.sql`, `src/migrate.py` | +| S3 | SeaweedFS S3 только HTTPS; minio-py с CA; разные ключи api/worker/janitor/admin (как в lab6), выдаются из OpenBao | `compose.yaml` (s3), `src/objstore.py` | +| Брокер | RabbitMQ: только AMQPS 5671, `verify_peer`; метрики только HTTPS 15691 c обязательным cert; epmd/dist — 127.0.0.1 | `infra/rabbitmq/rabbitmq.conf`, `src/common.py` | +| Телеметрия | OTLP/HTTPS + обязательный client cert на Collector; mTLS Collector → Tempo/Loki/Prometheus; scrape только HTTPS + cert; Grafana → datasources HTTPS + cert; Grafana сама — HTTPS | `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` | +| Пароли | **Argon2id** m=19456 KiB, t=2, p=1, соль 16 B, PHC-строка; процесс не стартует с параметрами ниже базовых; rehash scrypt/старых параметров при входе; dummy-hash для неизвестного логина | `src/security.py` | +| Alias реплики | `X-Instance-Id` = случайный `r-<10 hex>` на жизнь процесса (не hostname); `stop-one.sh` находит контейнер по `/tmp/instance-id` | `src/instance.py`, `scripts/stop-one.sh` | +| Hardening | Приложение: uid 10001, `cap_drop: ALL`, `no-new-privileges`, `read_only` + tmpfs `/tmp`, лимиты CPU/RAM/PIDs. Остальные: `no-new-privileges`, `cap_drop: ALL` (+ минимальный `cap_add` для entrypoint-ов db/broker/s3), read-only где совместимо. Наружу — только 127.0.0.1:8443 и 127.0.0.1:3000 | `compose.yaml` | + +**Сети.** `frontend` (lb, публикация), `apinet` internal (lb ↔ api), `backend` internal (данные), `telemetry` internal, `vault` internal (OpenBao + процессы приложения), `grafana-ui` (публикация Grafana). LB не видит БД/S3/брокер/OpenBao; Grafana — только telemetry. + +**Роли, сервисы и credentials** + +| Роль | Service | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | `haproxy:3.0.5-alpine` | `haproxy -c`; сам проверяет `/health/ready` API по mTLS | — | frontend+apinet+telemetry; `127.0.0.1:8443`; `:8404` (stats/metrics, mTLS, только telemetry) | Docker secrets: `lb-public.pem` (public-ca), `lb.pem` (identity lb) | +| API ×2 | `api` | `mtusi-messenger:lab7` | `127.0.0.1:8081/health/ready` (БД) | — | apinet+backend+telemetry+vault; `8443` mTLS, не опубликован | cert `api` (Docker secret) → OpenBao `kv/messenger/api`: пароль `api_svc`, ключ S3 api | +| Worker ×2 | `worker` | тот же | heartbeat-файл | tmpfs | backend+telemetry+vault | cert `worker` → `kv/messenger/worker`: `worker_svc`, ключ S3 worker, AMQPS user worker | +| Relay | `relay` | тот же | heartbeat | — | backend+telemetry+vault | cert `relay` → `relay_svc`, AMQPS user relay | +| Janitor | `janitor` | тот же | heartbeat | — | backend+telemetry+vault | cert `janitor` → `janitor_svc`, ключ S3 janitor | +| Миграции/роли | `migrate` (job) | тот же | Exited 0 | — | backend+vault | cert `migrate` → `kv/messenger/migrate` (суперпользователь + пароли ролей), токен 5 мин | +| Bucket | `s3-init` (job) | тот же | Exited 0 | — | backend+vault | cert `s3-init` → ключ S3 admin | +| PostgreSQL | `db` | `postgres:16.4-alpine` | `su-exec postgres pg_isready` | `pgdata` | backend; `5432` TLS | `postgres_password` (Docker secret), cert `db` | +| S3 | `s3` | `chrislusf/seaweedfs:4.50` | `curl --cacert … https://127.0.0.1:8333/healthz` + запись в filer | `s3data` | backend; `8333` HTTPS | `s3_identities.json` (Docker secret), cert `s3` | +| Брокер | `broker` | `rabbitmq:4.1.4-alpine` | running, alarms, listener 5671, очередь объявлена | `brokerdata` | backend+telemetry; `5671` AMQPS, `15691` HTTPS-метрики | `rabbitmq_definitions.json` (только хеши паролей), cert `broker` | +| Секреты | `openbao` | `openbao/openbao:2.7.2` | `bao status` (0 = unsealed) | `baodata` (Raft) | vault; `8200` TLS | cert `openbao`; root/shares — у оператора | +| Collector | `otel-collector` | `otel/opentelemetry-collector-contrib:0.163.2` | `up{job="otelcol-contrib"}` | — | telemetry; `4318` OTLP/HTTPS mTLS | cert `otel-collector` | +| Метрики | `prometheus` | `prom/prometheus:v3.13.5` | `promtool check ready` по mTLS | `promdata` | telemetry; `9090` mTLS | cert `prometheus` | +| Логи | `loki` | `grafana/loki:3.6.19` | `up{job="loki"}` | `lokidata` | telemetry; `3100` mTLS | cert `loki` | +| Трассы | `tempo` | `grafana/tempo:2.10.10` | `up{job="tempo"}` | `tempodata` | telemetry; `3200`, `4317` mTLS | cert `tempo` | +| UI | `grafana` | `grafana/grafana:12.4.14` | `curl --cacert public-ca https://localhost:3000/api/health` | `grafanadata` | telemetry+grafana-ui; `127.0.0.1:3000` HTTPS | admin — Docker secret; cert `grafana` (к datasources), `grafana-public` (сервер) | + +**Переменные `.env.example` → конфигурация.** `DATABASE_URL_FILE` skeleton → `POSTGRES_HOST/DB/USER` + пароль роли из OpenBao (`POSTGRES_PASSWORD`), `sslmode=verify-full`; `S3_ACCESS_KEY_ID_FILE/S3_SECRET_ACCESS_KEY_FILE` → поля KV `S3_ACCESS_KEY_ID/S3_SECRET_ACCESS_KEY`; `BROKER_URL_FILE` → поле KV `BROKER_URL` (amqps); `TLS_CA_FILE/TLS_CERT_FILE/TLS_KEY_FILE` → `/run/secrets/internal-ca.crt|service.crt|service.key` (те же имена, дефолты `src/tlsutil.py`); `BAO_ADDR/BAO_CACERT` → как в skeleton + `BAO_AUTH_ROLE`, `BAO_KV_PATH`; `ARGON2_*` — как в skeleton; `PUBLIC_BASE_URL` → `https://localhost:8443` (тесты); `APP_PORT=8443` (mTLS), `HEALTH_PORT=8081`. `SESSION_STORE=postgres`, `CACHE_URL_FILE` — не используются (Redis/Valkey нет). + +**Критичные зависимости readiness.** `/health/ready` API = PostgreSQL (как в lab6). OpenBao в readiness не входит: секреты читаются при старте процесса — без OpenBao процесс не стартует (и не становится ready), а работающий продолжает с секретами в памяти. Недоступность Grafana/телеметрии на API не влияет. ## Доказательства критериев +Все прогоны — на стенде выше, 2026-11-18, проект `mtusi-lab7`. Сводный прогон: `WITH_LAB6=1 ./student-tests/run_all.sh` → [evidence/run-all.txt](evidence/run-all.txt). + +**Воспроизводимость из чистого clone.** Копия репозитория без `secrets/`, `.env` и материала оператора (отдельный `COMPOSE_PROJECT_NAME=mtusi-lab7-clean` и `OPERATOR_DIR`): `init-secrets.sh` → `check_rules.sh` → `docker compose build --no-cache migrate` → `bao-init.sh` → `up --wait` → `WITH_LAB6=1 run_all.sh` — **ALL PASS** без ручных правок ([evidence/clean-clone-run.txt](evidence/clean-clone-run.txt)); затем этот стенд удалён (`down -v`). + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke по HTTPS | `make check`; `make test CA_FILE=…/public-ca.crt` | `PASS: lab7 public API scenario` (health → пользователи → вход → чат → сообщения → пагинация → ACL → идемпотентность → файл → thumbnail → logout) | evidence/make-test.txt | +| HTTPS с проверяемым CA и hostname/SAN; недоверенный TLS | `make test-security`; `lab7_api_security.py` (`tls.*`) | пустой trust store → `SSLCertVerificationError`; SNI `evil.example` → Hostname mismatch; internal-ca не валидирует ingress; TLS 1.1 → `PROTOCOL_VERSION` | evidence/make-test-security.txt, evidence/lab7-api-security.json | +| Argon2id: salt и параметры, не ниже 19 MiB/t=2/p=1, обоснование замером | `lab7_api_security.py` (`argon2.*`); `argon2_bench.py` в контейнере API | В БД `$argon2id$v=19$m=19456,t=2,p=1$$`, соли разные; rehash scrypt → Argon2id при входе; замер — раздел «Эксперименты» | evidence/lab7-api-security.json, evidence/argon2-bench.json | +| LB → каждая API-реплика: обязательное mTLS, проверка identity, нет обхода | `lab7_tls_matrix.py` (в сети стенда) | для обеих реплик: без cert → `CERTIFICATE_REQUIRED`; rogue CA (SAN lb) → `UNKNOWN_CA`; internal-ca, identity `intruder` → закрыто после handshake без HTTP-ответа; plaintext → reset; `:8081` из сети → refused; identity `lb` → 200, бизнес-API без Bearer → 401; порт API не опубликован | evidence/lab7-tls-matrix.json, evidence/lab7-hardening.json | +| TLS с проверкой сервера к DB, S3, broker, Collector, backends, OpenBao; раздельные credentials | `lab7_tls_matrix.py`; `selfcheck.py` внутри сервисов | 67/67: plaintext отвергается у всех (pg_hba reject, 400 HTTPS, refused), чужой CA/hostname → ошибка проверки, без client cert → отказ у Collector/Prometheus/Loki/Tempo/lb:8404/broker:15691; `pg_stat_ssl` TLSv1.3 у всех ролей; разные роли БД/ключи S3/пользователи брокера | evidence/lab7-tls-matrix.json, evidence/lab7-openbao.json | +| OpenBao не dev, persistent; сервисная аутентификация, scoped policy; нет root/unseal/широкого токена в приложении | `lab7_openbao.py` | `storage_type=raft`, `t=2 n=3`, entrypoint `bao server -config=…`; для api/worker/relay/janitor: policies `default,messenger-`, TTL 900, свой путь 200, чужой/`migrate`/list/запись/policy/token create/seal → 403, после revoke 403; в env нет токенов; смонтированы только CA+свой cert/key; KV пережил рестарт | evidence/lab7-openbao.json | +| Worker не получает admin credentials | `lab7_openbao.py` (`worker.*`) | ключ S3 worker = свой (не admin/api); `worker_svc` — denied на users/sessions/messages; путь `migrate` → 403 | evidence/lab7-openbao.json | +| Контейнеры: без privileged/host network/docker.sock; app/worker non-root, без capabilities, лимиты; наружу только TLS ingress и локальные admin | `lab7_hardening.py` | 182/182: no-new-privileges везде, `CapDrop ALL` везде (cap_add только db/broker/s3 — документировано), app uid 10001 + read-only rootfs, лимиты у всех; опубликованы только `127.0.0.1:8443` и `127.0.0.1:3000`; loopback health-listener изнутри контейнера отдаёт только `/health/*` (бизнес-API → 404) | evidence/lab7-hardening.json | +| Secret values/ключи не в Git, образах, telemetry | `scripts/scan_secrets.py`; `lab7_hardening.py` (`image.*`, `telemetry.*`); `obs_correlation.py` | 125 значений (пароли, ключи S3, тела PEM leaf и CA keys, root token, unseal shares) — 0 файлов; в образе нет `*.key/*.pem`/secrets, история слоёв чистая; в Loki/Tempo нет свежего токена/пароля (с положительным контролем поиска) | evidence/lab7-hardening.json, evidence/scan-secrets.txt, evidence/obs-correlation.json | +| Rate limit входа 429/Retry-After | `lab7_api_security.py` (`ratelimit.*`) | первый 429 на 21-й попытке за 10 с, `Retry-After: 10`, JSON `rate_limited` с `request_id`; после окна вход снова 201 | evidence/lab7-api-security.json | +| Ограничение размера запросов | `lab7_api_security.py` (`limits.*`) | JSON > 1 MiB → 413; upload > 10 MiB → 413 до чтения тела | evidence/lab7-api-security.json | +| Доступ постороннего; поддельный X-User-Id | `lab7_api_security.py` (`outsider.*`, `spoof.*`) | chat/history/metadata/content/thumbnail → 404; прикрепление чужого файла → 404; запись в чужой чат → 404; `X-User-Id` чужого пользователя не меняет `/users/me` | evidence/lab7-api-security.json | +| Logout / expiry | `lab7_api_security.py` (`logout.*`, `expiry.*`) | logout 204 → тот же токен 401, повтор 401; сессия с `expires_at` в прошлом (оператор, своя тестовая строка) → 401 | evidence/lab7-api-security.json | +| Сертификаты локально, документированный bootstrap | `./scripts/init-secrets.sh` | выпуск 2 CA + 19 leaf сервисов (+ тестовые: rogue CA, `intruder`, `rogue-lb`); CA keys в `$OPERATOR_DIR/ca` (600); README infra/openbao, trust-matrix | — | +| Перезапуск с сохранением данных (HTTPS) | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; хук перезапустил openbao (+unseal), db, s3, broker, prometheus, loki, tempo и все процессы — состояния до/после | evidence/test-persistence.txt, evidence/persistence-ps-before.txt, evidence/persistence-ps-after.txt | +| Последовательный отказ API-реплик (HTTPS) | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` ×N | PASS для api-1 и api-2 (для api-2 очередь round-robin сдвинута одним лишним запросом), 96–98 подтверждённых записей, 0 ошибок, восстановление записи 3.7–4.7 с | evidence/test-failover.txt | +| Нагрузка по HTTPS | `tests/load.py --duration 30 --concurrency 4` | 9588 успешных, 0 ошибок, 319.5 rps, p50 8.3 мс (после исправления Nagle; до — 80.9 rps, p50 48.8 мс) | evidence/load-after-nodelay.json, evidence/load-before-nodelay.json | +| Сохранение lab6 (наблюдаемость) поверх mTLS | `obs_correlation.py`, `dashboard_check.py`, `check_rules.sh` | трасса upload→relay→worker (26 spans), логи по trace_id в Loki; 3 datasource OK, 56 панелей без ошибок; все scrape-цели `up=1` | evidence/obs-correlation.json, evidence/dashboard-check.json, evidence/prometheus-up.txt | +| Сохранение lab5 (фоновая обработка) | `worker_outage.py` | остановка worker → upload 202 → рестарт API+брокера → ready, повторная доставка — no-op | evidence/worker-outage.txt | +| Модель угроз, trust matrix | — | 10 угроз с контролями и тестами; матрица заполнена, N/A с причинами | security/THREAT_MODEL.md, contracts/trust-matrix.md | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Пункты уровня 4, сделанные попутно (уровень 4 не заявляется): rehash старого хеша при входе; общий для реплик rate limit; read-only/no-new-privileges/drop capabilities и tmpfs; ограничение upload-worker по CPU/RAM/времени (cgroup + RLIMIT подпроцесса, с lab5); проверка отсутствия утечек в telemetry; поведение при sealed OpenBao. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Argon2id — выбор параметров** (`argon2_bench.py` внутри API-контейнера: cgroup 0.5 CPU, 640 MiB, aarch64; 15 хешей + 15 проверок на профиль, затем 2×15 параллельно — как `PASSWORD_HASH_CONCURRENCY=2`): + +| Профиль | hash p50 / p95, мс | verify p50, мс | рост RSS, MiB | хешей/с на реплику (2 потока) | +| --- | --- | --- | --- | --- | +| **m=19 MiB, t=2, p=1 (выбран)** | 13.0 / 64.8 | 13.2 | 18.2 | 27.2 | +| m=46 MiB, t=1, p=1 (OWASP alt) | 21.6 / 71.3 | 21.6 | 46.0 | 14.4 | +| m=19 MiB, t=3, p=1 | 23.4 / 108.6 | 69.7 | — | 16.6 | +| m=64 MiB, t=3, p=1 | 186.4 / 207.8 | 183.9 | — | 3.7 | +| m=19 MiB, t=2, p=2 | 8.0 / 80.8 | — | — | 15.0 | + +p95 завышен троттлингом CFS (квота 50 мс на 100 мс), это реальная задержка входа на стенде. Обоснование: базовый профиль OWASP (минимум задания) даёт ~13 мс на вход и ~27 входов/с на реплику (≈ 54/с на стенд); 2 одновременных хеша занимают ~37 MiB из 640 MiB. Профиль m=64 MiB/t=3 удорожает офлайн-перебор в ~14 раз, но снижает пропускную способность входа до 3.7/с на реплику — на 0.5 CPU 20 попыток (лимит LB за 10 с с одного IP) заняли бы ~5 с CPU всей реплики. Для учебного стенда с лимитом 0.5 CPU выбран базовый профиль; параметры меняются переменными `ARGON2_*` без миграции (хеши самоописаны, rehash при входе). + +**Нагрузка по HTTPS и найденная регрессия задержки.** `tests/load.py --duration 30 --concurrency 4` (closed-loop, только POST сообщений, клиент открывает новое TLS-соединение на каждый запрос; путь клиент →TLS→ LB →mTLS→ API → PostgreSQL TLS; телеметрия включена, 100 % трасс). Гипотеза: стабильные ~+40 мс к каждому ответу после перехода на TLS — эффект Nagle на сокете Python-сервера (заголовки и тело ответа — отдельные TLS-записи; вторая ждёт ACK первой, а HAProxy подтверждает с delayed ACK ~40 мс). Проверка — включить `TCP_NODELAY` на сокете API (`disable_nagle_algorithm = True`), остальное без изменений: + +| Конфигурация | успешных / ошибок | успешных rps | p50 / p95 / p99, мс | Файл | +| --- | --- | --- | --- | --- | +| lab6 (plaintext HTTP, тот же стенд) | 14173 / 0 | 471 | 4.8 / 37.6 / 56.3 | REPORT.md lab6 | +| lab7, Nagle включён | 2429 / 0 | 80.9 | 48.8 / 52.9 / 75.1 | evidence/load-before-nodelay.json | +| **lab7, TCP_NODELAY (итог)** | 9588 / 0 | **319.5** | **8.3** / 37.7 / 77.5 | evidence/load-after-nodelay.json | + +Гипотеза подтверждена (p50 −40 мс, rps ×4). Оставшаяся разница с lab6 (~32 %) — TLS-handshake на каждый запрос нагрузочного клиента (urllib без keep-alive) и шифрование на двух hop-ах; повторное использование соединений LB→API (`http-reuse safe`) сохраняется, TLS-сессии намеренно не переиспользуются (`no-ssl-reuse`, без tickets — каждое новое соединение заново проверяет identity). Ограничения: один прогон по 30 с на конфигурацию, closed-loop (coordinated omission), клиент на том же хосте; это сравнение конфигураций, а не предельная пропускная способность. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Рестарт всех хранилищ и процессов (`restart.sh`, вкл. OpenBao + unseal) | данные и прежняя сессия доступны | PASS `test-persistence` | ~60 с хук; потерь нет | +| Остановка api-1, затем api-2 (`stop-one.sh`) | запись продолжается через вторую реплику, без дублей | PASS ×2, 0 failed attempts | 3.7–4.7 с; потерь и дублей нет | +| OpenBao sealed (рестарт контейнера) при работающем API | работающие реплики обслуживают; новая не стартует; после unseal — стартует | вход и `/users/me` = 200; перезапущенная api-1 `unhealthy`, в логе `openbao not ready (… sealed)`; LB отдаёт ответы с другой реплики; после `bao-unseal.sh` — healthy | ~12 с после unseal (42 с от рестарта реплики); KV на месте | +| Чужой клиент в сети стенда (no cert / rogue CA / intruder) → API | отказ до HTTP | отказ во всех трёх случаях, событие `mtls.identity_rejected` | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Identity только на LB→API.** Collector, Prometheus, Loki, Tempo, `lb:8404` и RabbitMQ проверяют цепочку internal-ca, но не роль клиента: любой сервис с сертификатом internal-ca может слать телеметрию/читать метрики (тест фиксирует, что `intruder` принят Collector-ом). Брокер аутентифицирует по паролю + ACL, client cert не обязателен. Разграничение identity (OTel `auth` extension / sidecar, `rabbitmq_auth_mechanism_ssl`) — уровень 4. +- **Нет ротации и отзыва сертификатов** (CRL/OCSP, короткоживущие leaf из OpenBao PKI) и не продемонстрирована ротация секрета; механизм для паролей ролей БД есть (`bao-init.sh` + `migrate`). Leaf 30 дней — продление вручную. +- **Unseal shares** хранятся у оператора на том же хосте (отдельно от OpenBao, 600, вне проекта) — учебный компромисс; root token не отозван после bootstrap (нет отдельной операторской policy). +- **S3-права — по bucket**, не по prefix: worker и api читают все объекты bucket. +- **Rate limit — по IP на одном LB**: распределённый перебор не ограничен; один LB — единая точка отказа (как в lab3–6). +- Dependency/image scan с разбором применимости не выполнялся (уровень 4). +- Часть нагрузочных/отказных тестов lab4–5 (медленный клиент, прогрессивный upload, память upload, нагрузочный failover) написана под plaintext `:8080` и не перенесена на TLS — `student-tests/legacy/README.md`; обязательные возможности покрыты HTTPS smoke и перенесёнными проверками. +- Один Docker-host: отказ хоста, Docker daemon или root на хосте модель не покрывает. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2c083d3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,842 @@ +# Lab7 = lab6 + защита: HTTPS снаружи, mTLS LB->API, TLS ко всем зависимостям, OpenBao, hardening. +# +# клиент --HTTPS :8443 (public-ca)--> lb --mTLS (identity lb, internal-ca)--> api x2 +# api/relay/worker/janitor --TLS verify-full--> db (PostgreSQL, отдельные роли *_svc) +# --HTTPS--> s3 (SeaweedFS, отдельные ключи) +# relay/worker --AMQPS 5671--> broker (RabbitMQ, отдельные пользователи и ACL) +# все процессы приложения --OTLP/HTTPS + client cert--> otel-collector --mTLS--> prometheus/loki/tempo +# grafana --HTTPS + client cert--> prometheus/loki/tempo; оператор --HTTPS :3000--> grafana +# api/relay/worker/janitor/migrate/s3-init/ops --TLS + cert auth--> openbao (KV v2, scoped policy) +# +# Сети (все internal, кроме frontend и grafana-ui): apinet (lb <-> api), backend (данные), +# telemetry, vault (OpenBao только для процессов приложения). LB не видит БД/S3/брокер/OpenBao. +# На хост опубликованы ТОЛЬКО 127.0.0.1:8443 (HTTPS API) и 127.0.0.1:3000 (HTTPS Grafana). +# +# Запуск (подробно — REPORT.md): +# ./scripts/init-secrets.sh # пароли, PKI; CA keys -> каталог оператора +# ./scripts/bao-init.sh # ОПЕРАТОР: init/unseal OpenBao, policies, cert roles, KV +# docker compose --env-file .env up -d --build --wait +name: mtusi-lab7 + +x-app-image: &app-image + image: mtusi-messenger:lab7 + pull_policy: never + +# Hardening общий для всех сервисов: без повышения привилегий (setuid-бинарники бесполезны). +x-sec: &sec + security_opt: ["no-new-privileges:true"] + +# Процессы приложения: non-root (uid 10001 из образа), без capabilities, read-only rootfs, +# запись только в tmpfs /tmp (upload O_TMPFILE, рабочие файлы worker, heartbeat, instance-id). +x-app-sec: &app-sec + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + user: "10001:10001" + +x-otel-env: &otel-env + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + # Lab7: CA для проверки Collector и client cert сервиса (mTLS ingest). + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/internal-ca.crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/service.crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/service.key + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${APP_ENV:-lab} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_always_on} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-10000} + OTEL_EXPORTER_OTLP_TIMEOUT_SECONDS: ${OTEL_EXPORTER_OTLP_TIMEOUT_SECONDS:-5} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048} + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: ${OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED:-true} + +# Секреты сервисов — из OpenBao: cert auth (свой leaf) -> token со scoped policy -> KV v2. +x-bao-env: &bao-env + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/internal-ca.crt + BAO_KV_MOUNT: kv + BAO_STARTUP_WAIT_SECONDS: ${BAO_STARTUP_WAIT_SECONDS:-300} + +x-db-env: &db-env + POSTGRES_HOST: db + POSTGRES_PORT: "5432" + POSTGRES_DB: ${POSTGRES_DB:-messenger} + DB_SSLMODE: verify-full + +x-s3-env: &s3-env + S3_ENDPOINT: https://s3:8333 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-5} + +x-job-env: &job-env + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + +services: + # ===================================================================== secrets + # OpenBao НЕ в dev-mode: file storage на volume baodata, TLS-listener, после старта sealed. + # Init/unseal — оператор (scripts/bao-init.sh, scripts/bao-unseal.sh); unseal shares и root token + # лежат в каталоге оператора вне проекта и в контейнер не монтируются. + # healthcheck `bao status`: exit 0 только когда initialized + unsealed (sealed -> 2 -> unhealthy). + openbao: + image: openbao/openbao:2.7.2@sha256:5046911075e137e154ce49ab051502f4ec7cf367d484ae4a4dce665d9b90561f + entrypoint: ["bao", "server", "-config=/openbao/config/openbao.hcl"] + init: true + <<: *sec + cap_drop: [ALL] + read_only: true + environment: + BAO_ADDR: https://127.0.0.1:8200 + BAO_CACERT: /run/secrets/internal-ca.crt + SKIP_SETCAP: "true" + volumes: + - baodata:/openbao/file + - ./infra/openbao/openbao.hcl:/openbao/config/openbao.hcl:ro + tmpfs: ["/tmp:size=16m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: openbao_crt, target: service.crt} + - {source: openbao_key, target: service.key} + networks: [vault] + restart: unless-stopped + healthcheck: + test: ["CMD", "bao", "status"] + interval: 5s + start_interval: 1s + timeout: 3s + retries: 3 + start_period: 10s + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + + # ===================================================================== данные + # PostgreSQL: ssl=on, pg_hba только hostssl + scram-sha-256 (plaintext reject). Стартовый root + # нужен entrypoint-у образа (chown данных, копия ключа TLS с правами 0600), затем uid postgres. + db: + image: postgres:16.4-alpine@sha256:5660c2cbfea50c7a9127d17dc4e48543eedd3d7a41a595a2dfa572471e37e64c + entrypoint: ["/bin/sh", "/etc/postgresql/start.sh"] + <<: *sec + cap_drop: [ALL] + cap_add: [CHOWN, DAC_OVERRIDE, FOWNER, SETGID, SETUID] + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: postgres + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: "--auth-local=peer --auth-host=scram-sha-256" + secrets: + - postgres_password + - {source: db_crt, target: service.crt} + - {source: db_key, target: service.key} + volumes: + - pgdata:/var/lib/postgresql/data + - ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + - ./infra/postgres/start.sh:/etc/postgresql/start.sh:ro + networks: [backend] + restart: unless-stopped + healthcheck: + # от имени ОС-пользователя postgres (локальный сокет, peer): под root был бы «Peer authentication failed» + test: ["CMD-SHELL", "su-exec postgres pg_isready -q -U postgres -d \"$${POSTGRES_DB}\""] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + + # S3: SeaweedFS, S3 API только HTTPS :8333 (сертификат s3). master/volume/filer — 127.0.0.1 + # контейнера (закрытый локальный hop, снаружи недостижим). + s3: + image: chrislusf/seaweedfs:4.50@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860 + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -master.volumeSizeLimitMB=1024 + - -master.electionTimeout=2s + - -volume.preStopSeconds=2 + - -volume.max=0 + - -s3 + - -s3.ip.bind=0.0.0.0 + - -s3.port=8333 + - -s3.cert.file=/run/secrets/service.crt + - -s3.key.file=/run/secrets/service.key + - -s3.config=/run/secrets/s3_identities.json + - -s3.iam=false + - -s3.port.iceberg=0 + - -s3.port.lance=0 + - -s3.concurrentUploadLimitMB=64 + - -filer.concurrentUploadLimitMB=64 + - -volume.concurrentUploadLimitMB=64 + - -s3.cacheCapacityMB=16 + - -s3.readerCacheSizeMB=32 + # entrypoint образа: chown /data и su-exec на пользователя seaweed (дальше процесс не root). + <<: *sec + cap_drop: [ALL] + cap_add: [CHOWN, SETGID, SETUID] + secrets: + - s3_identities.json + - {source: internal_ca, target: internal-ca.crt} + - {source: s3_crt, target: service.crt} + - {source: s3_key, target: service.key} + environment: + GOMEMLIMIT: 512MiB + volumes: + - s3data:/data + networks: [backend] + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -sf -m 2 --cacert /run/secrets/internal-ca.crt https://127.0.0.1:8333/healthz -o /dev/null && curl -sf -m 2 -X PUT --data-binary ok http://127.0.0.1:8888/.healthcheck/probe -o /dev/null"] + interval: 5s + start_interval: 1s + timeout: 3s + retries: 3 + start_period: 60s + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.0", memory: 768M} + + # RabbitMQ: только AMQPS 5671 и HTTPS-метрики 15691 (client cert обязателен). Узел rabbit@localhost: + # epmd и Erlang distribution слушают только 127.0.0.1 контейнера (кластерного канала нет). + broker: + image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2 + hostname: broker + <<: *sec + cap_drop: [ALL] + cap_add: [CHOWN, DAC_OVERRIDE, FOWNER, SETGID, SETUID] + environment: + RABBITMQ_NODENAME: rabbit@localhost + ERL_EPMD_ADDRESS: 127.0.0.1 + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-kernel inet_dist_use_interface {127,0,0,1}" + volumes: + - brokerdata:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab7.conf:ro + secrets: + - {source: rabbitmq_definitions.json, target: rabbitmq_definitions.json} + - {source: internal_ca, target: internal-ca.crt} + - {source: broker_crt, target: service.crt} + - {source: broker_key, target: service.key} + networks: [backend, telemetry] + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms && rabbitmq-diagnostics -q check_port_listener 5671 && rabbitmqctl -q -p messenger list_queues name | grep -q attachments.process.v1"] + interval: 10s + start_interval: 2s + timeout: 15s + retries: 5 + start_period: 60s + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + + # ===================================================================== bootstrap jobs + migrate: + <<: [*app-image, *app-sec] + build: + context: . + args: + APP_VERSION: ${APP_VERSION:-1} + pull_policy: missing + command: ["python", "/app/src/migrate.py"] + environment: + <<: [*db-env, *bao-env] + POSTGRES_USER: postgres + # migrate импортирует server.py (STORE/MIGRATIONS_DIR); S3-клиент создаётся, но не используется. + S3_ENDPOINT: https://s3:8333 + S3_CLIENT_REQUIRED: "0" + BAO_AUTH_ROLE: migrate + BAO_KV_PATH: messenger/migrate + tmpfs: ["/tmp:size=8m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: migrate_crt, target: service.crt} + - {source: migrate_key, target: service.key} + networks: [backend, vault] + healthcheck: {disable: true} + restart: "no" + depends_on: + db: {condition: service_healthy} + openbao: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + + s3-init: + <<: [*app-image, *app-sec] + command: ["python", "/app/src/s3_bootstrap.py"] + environment: + <<: [*s3-env, *bao-env] + BAO_AUTH_ROLE: s3-init + BAO_KV_PATH: messenger/s3-init + tmpfs: ["/tmp:size=8m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: s3_init_crt, target: service.crt} + - {source: s3_init_key, target: service.key} + networks: [backend, vault] + healthcheck: {disable: true} + restart: "no" + depends_on: + s3: {condition: service_healthy} + openbao: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + + # ===================================================================== приложение + api: + <<: [*app-image, *app-sec] + expose: ["8443"] + environment: + <<: [*db-env, *s3-env, *job-env, *otel-env, *bao-env] + POSTGRES_USER: api_svc + BAO_AUTH_ROLE: api + BAO_KV_PATH: messenger/api + APP_HOST: 0.0.0.0 + APP_PORT: "8443" + MTLS_ALLOWED_CLIENTS: lb + HEALTH_HOST: 127.0.0.1 + HEALTH_PORT: "8081" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456} + ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2} + ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1} + DB_POOL_SIZE: ${DB_POOL_SIZE:-10} + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-10} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2} + PASSWORD_HASH_WAIT_SECONDS: ${PASSWORD_HASH_WAIT_SECONDS:-0.5} + DB_HEALTH_INTERVAL_SECONDS: ${DB_HEALTH_INTERVAL_SECONDS:-1} + READY_STALE_SECONDS: ${READY_STALE_SECONDS:-3} + UPLOAD_TIMEOUT_SECONDS: ${UPLOAD_TIMEOUT_SECONDS:-60} + JSON_BODY_TIMEOUT_SECONDS: ${JSON_BODY_TIMEOUT_SECONDS:-10} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8} + DECODE_CONCURRENCY: ${DECODE_CONCURRENCY:-1} + JPEG_PROBE_MEMORY_MB: ${JPEG_PROBE_MEMORY_MB:-384} + JPEG_PROBE_CPU_SECONDS: ${JPEG_PROBE_CPU_SECONDS:-10} + JPEG_PROBE_TIMEOUT_SECONDS: ${JPEG_PROBE_TIMEOUT_SECONDS:-15} + S3_PUT_CONCURRENCY: ${S3_PUT_CONCURRENCY:-3} + FAULT_INJECTION: ${FAULT_INJECTION:-0} + FAULT_SLOW_HANDLER_SECONDS: ${FAULT_SLOW_HANDLER_SECONDS:-1.5} + OTEL_SERVICE_NAME: messenger-api + tmpfs: ["/tmp:size=160m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: api_crt, target: service.crt} + - {source: api_key, target: service.key} + # У API нет доступа к брокеру (задание пишется в outbox PostgreSQL). + networks: [apinet, backend, telemetry, vault] + restart: unless-stopped + stop_grace_period: 20s + depends_on: + db: {condition: service_healthy} + openbao: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "0.5", memory: 640M, pids: 256} + + relay: + <<: [*app-image, *app-sec] + command: ["python", "/app/src/relay.py"] + environment: + <<: [*db-env, *job-env, *otel-env, *bao-env] + POSTGRES_USER: relay_svc + BAO_AUTH_ROLE: relay + BAO_KV_PATH: messenger/relay + SERVICE_NAME: relay + RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-1} + REAPER_INTERVAL_SECONDS: ${REAPER_INTERVAL_SECONDS:-5} + JOB_REDRIVE_AFTER_SECONDS: ${JOB_REDRIVE_AFTER_SECONDS:-900} + OUTBOX_RETENTION_SECONDS: ${OUTBOX_RETENTION_SECONDS:-86400} + RELAY_FAULT: ${RELAY_FAULT:-} + OTEL_SERVICE_NAME: messenger-relay + tmpfs: ["/tmp:size=8m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: relay_crt, target: service.crt} + - {source: relay_key, target: service.key} + networks: [backend, telemetry, vault] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/relay.heartbeat') < 30 else 1)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 30s + restart: unless-stopped + stop_grace_period: 15s + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + openbao: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.25", memory: 128M, pids: 64} + + # Upload-worker: ограничен по CPU/RAM (cgroup) и по времени/памяти задания (подпроцесс transform: + # RLIMIT_AS/RLIMIT_CPU + JOB_TIMEOUT_SECONDS). + worker: + <<: [*app-image, *app-sec] + command: ["python", "/app/src/worker.py"] + environment: + <<: [*db-env, *s3-env, *job-env, *otel-env, *bao-env] + POSTGRES_USER: worker_svc + BAO_AUTH_ROLE: worker + BAO_KV_PATH: messenger/worker + SERVICE_NAME: worker + S3_READ_TIMEOUT_SECONDS: ${WORKER_S3_READ_TIMEOUT_SECONDS:-10} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + TRANSFORM_MEMORY_LIMIT_MB: ${TRANSFORM_MEMORY_LIMIT_MB:-768} + TRANSFORM_CPU_SECONDS: ${TRANSFORM_CPU_SECONDS:-30} + WORKER_TMP_DIR: /tmp/work + WORKER_FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-0} + FAULT_SLOW_SECONDS: ${FAULT_SLOW_SECONDS:-20} + OTEL_SERVICE_NAME: messenger-worker + tmpfs: ["/tmp:size=96m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: worker_crt, target: service.crt} + - {source: worker_key, target: service.key} + networks: [backend, telemetry, vault] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/worker.heartbeat') < 30 else 1)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 30s + restart: unless-stopped + stop_grace_period: 75s + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + broker: {condition: service_healthy} + openbao: {condition: service_healthy} + deploy: + replicas: ${WORKER_REPLICAS:-2} + resources: + limits: {cpus: "1.0", memory: 1G, pids: 128} + + janitor: + <<: [*app-image, *app-sec] + command: ["python", "/app/src/janitor.py"] + environment: + <<: [*db-env, *s3-env, *otel-env, *bao-env] + POSTGRES_USER: janitor_svc + BAO_AUTH_ROLE: janitor + BAO_KV_PATH: messenger/janitor + ATTACHMENT_GRACE_SECONDS: ${ATTACHMENT_GRACE_SECONDS:-3600} + JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-300} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + OTEL_SERVICE_NAME: messenger-janitor + tmpfs: ["/tmp:size=8m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: janitor_crt, target: service.crt} + - {source: janitor_key, target: service.key} + networks: [backend, telemetry, vault] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/janitor.heartbeat') < 90 else 1)"] + interval: 15s + timeout: 3s + retries: 2 + start_period: 20s + restart: unless-stopped + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + openbao: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.25", memory: 160M, pids: 64} + + # Ingress: HTTPS :8443 наружу (только 127.0.0.1 хоста), mTLS к API. Не видит БД/S3/OpenBao. + lb: + image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949 + <<: *sec + cap_drop: [ALL] + read_only: true + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + environment: + LOGIN_RATE_LIMIT: ${LOGIN_RATE_LIMIT:-20} + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: lb_public_pem, target: lb-public.pem} + - {source: lb_pem, target: lb.pem} + ports: + - "127.0.0.1:${LB_PORT:-8443}:8443" + networks: [frontend, apinet, telemetry] + restart: unless-stopped + healthcheck: + test: ["CMD", "haproxy", "-c", "-q", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + depends_on: + api: {condition: service_started} + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + + # Операторский CLI: docker compose --profile tools run --rm ops <команда> + ops: + <<: [*app-image, *app-sec] + profiles: [tools] + entrypoint: ["python", "/app/src/ops.py"] + environment: + <<: [*db-env, *s3-env, *job-env, *bao-env] + POSTGRES_USER: ops_svc + BAO_AUTH_ROLE: ops + BAO_KV_PATH: messenger/ops + SERVICE_NAME: ops + tmpfs: ["/tmp:size=8m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: ops_crt, target: service.crt} + - {source: ops_key, target: service.key} + networks: [backend, vault] + healthcheck: {disable: true} + restart: "no" + + # Проверки ВНУТРИ стенда (отрицательные mTLS/TLS-тесты, права S3, варианты). Только для тестов: + # получает тестовые identities (intruder, rogue-lb) и — для положительной проверки mTLS — lb. + checks: + <<: [*app-image, *app-sec] + profiles: [tools] + entrypoint: ["python"] + environment: + <<: *s3-env + PYTHONDONTWRITEBYTECODE: "1" + tmpfs: ["/tmp:size=64m,mode=1777"] + volumes: + - ./student-tests:/app/student-tests:ro + - ./tests:/app/tests:ro + - ./contracts:/app/contracts:ro + - ./lab.json:/app/lab.json:ro + - ./evidence:/app/evidence + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: public_ca, target: public-ca.crt} + - {source: lb_crt, target: lb.crt} + - {source: lb_key, target: lb.key} + - {source: intruder_crt, target: intruder.crt} + - {source: intruder_key, target: intruder.key} + - {source: rogue_ca, target: rogue-ca.crt} + - {source: rogue_lb_crt, target: rogue-lb.crt} + - {source: rogue_lb_key, target: rogue-lb.key} + - s3_api_access_key + - s3_api_secret_key + - s3_worker_access_key + - s3_worker_secret_key + - s3_janitor_access_key + - s3_janitor_secret_key + networks: [apinet, backend, telemetry, vault] + healthcheck: {disable: true} + restart: "no" + + # ===================================================================== телеметрия (lab6 + TLS) + otel-collector: + image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849 + command: ["--config=/etc/otelcol/collector.yaml"] + <<: *sec + cap_drop: [ALL] + read_only: true + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: otel_collector_crt, target: service.crt} + - {source: otel_collector_key, target: service.key} + environment: + GOMEMLIMIT: 200MiB + networks: [telemetry] + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + + prometheus: + image: prom/prometheus:v3.13.5@sha256:aafa9d694e6a883c02db516d9f6e5b84e14195f49a2a6456a4cca815edda72b9 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.config.file=/etc/prometheus/web.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --storage.tsdb.retention.size=2GB + - --web.enable-otlp-receiver + - --enable-feature=exemplar-storage + - --web.listen-address=:9090 + <<: *sec + cap_drop: [ALL] + read_only: true + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro + - ./infra/prometheus/client.yml:/etc/prometheus/client.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - promdata:/prometheus + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: prometheus_crt, target: service.crt} + - {source: prometheus_key, target: service.key} + networks: [telemetry] + restart: unless-stopped + healthcheck: + test: ["CMD", "promtool", "check", "ready", "--url=https://prometheus:9090", "--http.config.file=/etc/prometheus/client.yml"] + interval: 10s + start_interval: 2s + timeout: 3s + retries: 5 + start_period: 30s + deploy: + resources: + limits: {cpus: "0.5", memory: 512M} + + loki: + image: grafana/loki:3.6.19@sha256:0f9cbc340cbe33ee498e654075034336962057db072f57b37f253e14d476443a + command: ["-config.file=/etc/loki/loki.yaml"] + <<: *sec + cap_drop: [ALL] + read_only: true + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + tmpfs: ["/tmp:size=32m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: loki_crt, target: service.crt} + - {source: loki_key, target: service.key} + networks: [telemetry] + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.5", memory: 512M} + + tempo: + image: grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38 + command: ["-config.file=/etc/tempo/tempo.yaml"] + <<: *sec + cap_drop: [ALL] + read_only: true + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + tmpfs: ["/tmp:size=32m,mode=1777"] + secrets: + - {source: internal_ca, target: internal-ca.crt} + - {source: tempo_crt, target: service.crt} + - {source: tempo_key, target: service.key} + networks: [telemetry] + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.5", memory: 512M} + + # Grafana: HTTPS (public-ca) на 127.0.0.1:3000; к datasources — HTTPS + client cert grafana. + grafana: + image: grafana/grafana:12.4.14@sha256:ae509d9ae8c1021ca32ec370a9ca975d40165eda9fccd1e1102d317ff8787cd7 + <<: *sec + cap_drop: [ALL] + environment: + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /run/secrets/public.crt + GF_SERVER_CERT_KEY: /run/secrets/public.key + GF_SERVER_MIN_TLS_VERSION: TLS1.2 + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_SECURITY_COOKIE_SECURE: "true" + GF_SECURITY_COOKIE_SAMESITE: strict + GF_SECURITY_DISABLE_GRAVATAR: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + secrets: + - grafana_admin_password + - {source: internal_ca, target: internal-ca.crt} + - {source: public_ca, target: public-ca.crt} + - {source: grafana_crt, target: service.crt} + - {source: grafana_key, target: service.key} + - {source: grafana_public_crt, target: public.crt} + - {source: grafana_public_key, target: public.key} + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + networks: [telemetry, grafana-ui] + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-sf", "-m", "2", "--cacert", "/run/secrets/public-ca.crt", "https://localhost:3000/api/health", "-o", "/dev/null"] + interval: 10s + start_interval: 2s + timeout: 3s + retries: 6 + start_period: 60s + depends_on: + prometheus: {condition: service_started} + loki: {condition: service_started} + tempo: {condition: service_started} + deploy: + resources: + limits: {cpus: "0.5", memory: 384M} + +secrets: + internal_ca: + file: ./secrets/pki/internal-ca.crt + public_ca: + file: ./secrets/pki/public-ca.crt + lb_public_pem: + file: ./secrets/pki/lb-public.pem + lb_pem: + file: ./secrets/pki/lb.pem + grafana_public_crt: + file: ./secrets/pki/grafana-public.crt + grafana_public_key: + file: ./secrets/pki/grafana-public.key + lb_crt: + file: ./secrets/pki/lb.crt + lb_key: + file: ./secrets/pki/lb.key + api_crt: + file: ./secrets/pki/api.crt + api_key: + file: ./secrets/pki/api.key + worker_crt: + file: ./secrets/pki/worker.crt + worker_key: + file: ./secrets/pki/worker.key + relay_crt: + file: ./secrets/pki/relay.crt + relay_key: + file: ./secrets/pki/relay.key + janitor_crt: + file: ./secrets/pki/janitor.crt + janitor_key: + file: ./secrets/pki/janitor.key + migrate_crt: + file: ./secrets/pki/migrate.crt + migrate_key: + file: ./secrets/pki/migrate.key + s3_init_crt: + file: ./secrets/pki/s3-init.crt + s3_init_key: + file: ./secrets/pki/s3-init.key + ops_crt: + file: ./secrets/pki/ops.crt + ops_key: + file: ./secrets/pki/ops.key + grafana_crt: + file: ./secrets/pki/grafana.crt + grafana_key: + file: ./secrets/pki/grafana.key + db_crt: + file: ./secrets/pki/db.crt + db_key: + file: ./secrets/pki/db.key + s3_crt: + file: ./secrets/pki/s3.crt + s3_key: + file: ./secrets/pki/s3.key + broker_crt: + file: ./secrets/pki/broker.crt + broker_key: + file: ./secrets/pki/broker.key + otel_collector_crt: + file: ./secrets/pki/otel-collector.crt + otel_collector_key: + file: ./secrets/pki/otel-collector.key + prometheus_crt: + file: ./secrets/pki/prometheus.crt + prometheus_key: + file: ./secrets/pki/prometheus.key + loki_crt: + file: ./secrets/pki/loki.crt + loki_key: + file: ./secrets/pki/loki.key + tempo_crt: + file: ./secrets/pki/tempo.crt + tempo_key: + file: ./secrets/pki/tempo.key + openbao_crt: + file: ./secrets/pki/openbao.crt + openbao_key: + file: ./secrets/pki/openbao.key + # тестовые identities (только сервис checks) + intruder_crt: + file: ./secrets/pki/test/intruder.crt + intruder_key: + file: ./secrets/pki/test/intruder.key + rogue_ca: + file: ./secrets/pki/test/rogue-ca.crt + rogue_lb_crt: + file: ./secrets/pki/test/rogue-lb.crt + rogue_lb_key: + file: ./secrets/pki/test/rogue-lb.key + # серверная сторона хранилищ и тестовые ключи S3 + postgres_password: + file: ./secrets/postgres_password + grafana_admin_password: + file: ./secrets/grafana_admin_password + s3_identities.json: + file: ./secrets/s3_identities.json + rabbitmq_definitions.json: + file: ./secrets/rabbitmq_definitions.json + s3_api_access_key: + file: ./secrets/s3_api_access_key + s3_api_secret_key: + file: ./secrets/s3_api_secret_key + s3_worker_access_key: + file: ./secrets/s3_worker_access_key + s3_worker_secret_key: + file: ./secrets/s3_worker_secret_key + s3_janitor_access_key: + file: ./secrets/s3_janitor_access_key + s3_janitor_secret_key: + file: ./secrets/s3_janitor_secret_key + +networks: + frontend: {} + apinet: + internal: true + backend: + internal: true + telemetry: + internal: true + vault: + internal: true + grafana-ui: {} + +volumes: + baodata: {} + pgdata: {} + s3data: {} + brokerdata: {} + promdata: {} + lokidata: {} + tempodata: {} + grafanadata: {} diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..53b3021 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,57 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия — решение lab7 (Лазарев Ю. С., БПИ2303) -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +Это обязательная матрица покрытия. Для каждой строки указаны реальный порт, endpoint, CA, service identity, проверка SAN/ACL, срок жизни credentials и команда положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. + +**Обозначения.** `public-ca` — CA внешнего ingress (его сертификат раздаётся клиентам: `secrets/pki/public-ca.crt`). `internal-ca` — CA сервисов (mTLS/TLS внутри стенда). Закрытые ключи обоих CA — только в каталоге оператора `$OPERATOR_DIR/ca/` (вне проекта, не монтируются в контейнеры). Leaf-сертификаты: ECDSA P-256, **30 дней**, SAN DNS = имя сервиса Compose (это и есть identity); выпуск — `scripts/init-secrets.sh` → `scripts/pki-issue.sh`. Каждый контейнер получает Docker secret-ами **только свой** `service.crt/service.key` + `internal-ca.crt`. + +Тесты: **M** = `docker compose --profile tools run --rm -T checks /app/student-tests/lab7_tls_matrix.py` (67 проверок, [evidence/lab7-tls-matrix.json](../evidence/lab7-tls-matrix.json)); **A** = `python3 student-tests/lab7_api_security.py` ([evidence/lab7-api-security.json](../evidence/lab7-api-security.json)); **B** = `python3 student-tests/lab7_openbao.py` ([evidence/lab7-openbao.json](../evidence/lab7-openbao.json)); **H** = `python3 student-tests/lab7_hardening.py`; **S** = `make test-security`. | Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Клиент → ingress/LB | **Сделано.** `127.0.0.1:8443` → HAProxy `bind :8443 ssl crt lb-public.pem` (public-ca, SAN `localhost`, `127.0.0.1`), TLS ≥ 1.2, без session tickets, HSTS. Клиент проверяет CA + hostname; пользователь — Bearer (сессия в PostgreSQL, TTL 3600 с, отзыв `DELETE /auth/sessions/current`). Rate limit входа на LB (stick-table, 20/10 с с IP → 429 + `Retry-After: 10`), общий для всех реплик. | Expiry/revocation сессии — проверены (A: `expiry.*`, `logout.*`); общий login rate limit — есть (LB). Ротация leaf ingress — не выполнялась (уровень 4 не заявлен). | Конфиг: [infra/haproxy/haproxy.cfg](../infra/haproxy/haproxy.cfg) `frontend fe_api`. **+** `make test CA_FILE=…/public-ca.crt` (smoke PASS). **−** S: пустой trust store → `SSLCertVerificationError`; A: `tls.wrong_hostname_rejected` (SNI `evil.example` → Hostname mismatch), `tls.internal_ca_does_not_validate_ingress`, `tls.tls11_refused` (alert protocol_version); `ratelimit.login_429_with_retry_after`. | +| LB → API (все реплики) | **Сделано: обязательное mTLS + allowlist identity.** HAProxy `server-template api 8 api:8443 ssl verify required ca-file internal-ca.crt verifyhost api crt lb.pem no-ssl-reuse` (LB проверяет SAN `api`, предъявляет identity `lb`). API (`src/server.py`, `src/tlsutil.py`): `CERT_REQUIRED` + internal-ca, затем **SAN DNS клиента ∈ `MTLS_ALLOWED_CLIENTS=lb`**, иначе соединение закрывается до разбора HTTP (событие `mtls.identity_rejected`, метрика `messenger.mtls.rejected`). Без session tickets на API → каждое новое соединение проходит проверку. Health-проверки LB идут тем же mTLS. Порт API наружу не опубликован; сеть `apinet` internal (только lb и api). Health для Docker — plaintext **только 127.0.0.1:8081** внутри контейнера, отдаёт лишь `/health/*`. | Ротация leaf без обхода — не выполнялась. | **+** M: `identity lb -> /health/ready` = 200 для каждой реплики, `identity lb, no Bearer -> business API` = 401 (mTLS не заменяет сессию). **−** M: без client cert → `TLSV13_ALERT_CERTIFICATE_REQUIRED`; cert rogue CA с SAN `lb` → `UNKNOWN_CA`; доверенный CA, identity `intruder` → `closed_after_handshake_no_http_response`; plaintext → reset; `:8081` из сети → refused; ожидание чужого hostname → Hostname mismatch. H: порт API не опубликован. | +| API/worker → PostgreSQL | **Сделано.** `db:5432`, `ssl=on`, сертификат `db` (internal-ca); клиенты `sslmode=verify-full sslrootcert=internal-ca.crt`. [pg_hba.conf](../infra/postgres/pg_hba.conf): только `hostssl … scram-sha-256`, `hostnossl … reject`. Раздельные роли: `api_svc`, `worker_svc`, `relay_svc`, `janitor_svc`, `ops_svc` (NOSUPERUSER, NOCREATEROLE, NOCREATEDB, NOBYPASSRLS, CONNECTION LIMIT) с грантами по таблицам ([grants.sql](../migrations/security/grants.sql)); DDL и владение — только `postgres` (job `migrate`). Пароли ролей — из OpenBao (KV), синхронизируются job `migrate`. | Ротация пароля роли — механизм есть (повторный `migrate` выставляет пароль из KV), сценарий не демонстрировался; mTLS к БД — нет. | **+** B/selfcheck: `pg_stat_ssl` = TLSv1.3 у каждой роли; M: `verify-full, internal CA, wrong pwd` → TLS ok, SCRAM отверг пароль. **−** M: `sslmode=disable` → `pg_hba.conf rejects … no encryption`; rogue CA → cert verify failed; подключение по IP → SAN mismatch. B: worker `SELECT users/sessions/messages` → denied; api `CREATE TABLE`/`DROP`/`TRUNCATE`/`UPDATE chats.title` → denied. | +| API/worker → S3 | **Сделано.** `https://s3:8333` (SeaweedFS `-s3.cert.file/-s3.key.file`, сертификат `s3`); клиенты minio-py с `cert_reqs=CERT_REQUIRED, ca_certs=internal-ca` (http:// endpoint процесс отвергает). Ключи разные: `api` (Read+Write bucket), `worker` (Read+Write), `janitor` (List+Write, без чтения содержимого), `admin` (только job `s3-init`); приложения получают свои ключи из OpenBao. master/volume/filer SeaweedFS слушают только 127.0.0.1 контейнера (закрытый локальный hop). Bucket приватный. | Права по prefix и ротация ключей — не делались (ограничение по bucket). | **+** smoke (upload/download по HTTPS), M: анонимный list по проверенному TLS → 403. **−** M: plaintext → `400 … HTTPS`; rogue CA → verify failed. B: у worker ключ S3 = ключ `worker` (SHA-префикс), не `admin`/`api`. Конфиг: compose `s3`, `secrets/s3_identities.json` (генерируется init-secrets). | +| API/worker → broker | **Сделано (на 3).** RabbitMQ: `listeners.tcp = none`, только AMQPS `broker:5671` (сертификат `broker`, TLS 1.2/1.3), `verify_peer` (предъявленный client cert проверяется), клиенты pika с CA + `server_hostname=broker` и своим client cert. Отдельные пользователи и ACL vhost `messenger`: `relay` — только write в exchange `attachments`; `worker` — только read из `attachments.process.v1`; `ops` — DLQ; ни у кого нет configure. **API к брокеру доступа не имеет** (outbox в PostgreSQL). Erlang distribution/epmd — только 127.0.0.1 (`rabbit@localhost`). | mTLS с обязательным cert и проверкой SAN/роли — не реализовано (уровень 4). | [infra/rabbitmq/rabbitmq.conf](../infra/rabbitmq/rabbitmq.conf). **+** M: AMQPS с проверкой CA → Connection.Start; smoke (worker обработал файл). **−** M: plaintext 5672 → refused; rogue CA trust → verify failed; client cert rogue CA → `UNKNOWN_CA`; `worker` с неверным паролем → auth refused. | +| API/worker → Collector | **Сделано: TLS + обязательный client cert (mTLS по CA).** `https://otel-collector:4318` (OTLP/HTTP), `client_ca_file: internal-ca` — без сертификата стенда ingest невозможен; Python SDK: `OTEL_EXPORTER_OTLP_CERTIFICATE/CLIENT_CERTIFICATE/CLIENT_KEY`. OTLP gRPC 4317, метрики 8888 и health 13133 Collector-а — только 127.0.0.1. Сеть `telemetry` internal. | Раздельные identities: сертификаты уже раздельные (SAN = сервис), но **Collector не ограничивает роль** — `intruder` (internal-ca) принят (M фиксирует это как известное ограничение уровня 3). | [infra/otel/collector.yaml](../infra/otel/collector.yaml). **+** `obs_correlation.py` (трасса API→relay→worker, логи в Loki). **−** M: plaintext → 400; без client cert → `CERTIFICATE_REQUIRED`; rogue CA → `UNKNOWN_CA`; `:4317/:8888/:13133` из сети → refused. | +| Collector → telemetry backends | **Сделано: mTLS.** Tempo OTLP gRPC `tempo:4317` (`tls.client_ca_file`), Loki `https://loki:3100/otlp`, Prometheus `https://prometheus:9090/api/v1/otlp` (`web.yml`: `RequireAndVerifyClientCert`); Collector проверяет CA + `server_name_override` и предъявляет `otel-collector`. Внутренний gRPC Loki (9095) и Tempo (9096) — 127.0.0.1. Scrape Prometheus → HAProxy `lb:8404`, RabbitMQ `broker:15691`, Loki, Tempo, сам Prometheus — только HTTPS + client cert `prometheus`; собственные метрики Collector идут через его же pipeline (receiver `prometheus/self` с loopback), plaintext scrape убран. | Ротация, least privilege по identity — нет (CA-only). | [infra/prometheus/web.yml](../infra/prometheus/web.yml), [prometheus.yml](../infra/prometheus/prometheus.yml), [loki.yaml](../infra/loki/loki.yaml), [tempo.yaml](../infra/tempo/tempo.yaml). **+** `up` = 1 у всех целей ([evidence/prometheus-up.txt](../evidence/prometheus-up.txt)). **−** M: для prometheus/loki/tempo/tempo-gRPC/lb:8404 — plaintext, без cert, rogue CA, чужой hostname → отказ; broker 15692 из сети → refused, 15691 без cert → отказ. | +| Grafana → data sources | **Сделано.** Datasources `https://prometheus:9090`, `https://loki:3100`, `https://tempo:3200`, `tlsAuthWithCACert` (internal-ca), `serverName`, `tlsSkipVerify: false`, client cert `grafana` через `$__file{}` из Docker secret (в Git ключа нет). | Отрицательные проверки доступа по identity — нет (backends CA-only). | [datasources.yaml](../infra/grafana/provisioning/datasources/datasources.yaml). **+** `dashboard_check.py`: datasource health OK ×3, 56 панелей без ошибок. **−** те же M-проверки backend-ов без/с чужим cert. | +| API/worker/agent → OpenBao | **Сделано.** `https://openbao:8200` (TLS 1.2+, сертификат `openbao`), сеть `vault` internal (только процессы приложения). Аутентификация — **auth/cert**: роль на сервис (`allowed_dns_sans=<сервис>`, `certificate=internal-ca`), policy `messenger-<сервис>` = `read` только `kv/data/messenger/<сервис>`; токен service, TTL 15 мин, max 1 ч (`migrate`/`s3-init` — 5/10 мин); фоновое renew-self на ½ TTL, при неудаче — повторный login. Root token, unseal shares, `BAO_TOKEN`, SecretID в приложении отсутствуют. | Renewal есть и проверен (`renew_self`=200), revoke-self → 403; expiry/ротация секрета под нагрузкой — не демонстрировались. | [infra/openbao/](../infra/openbao/), [scripts/bao-init.sh](../scripts/bao-init.sh), [src/vault.py](../src/vault.py). **+** B: свой путь 200, policies `default,messenger-`, TTL ≤ 900. **−** B: чужой сервис / admin-путь `migrate` / list metadata / запись секрета / запись policy / `auth/token/create` / `sys/seal` / list ролей → 403; токен после revoke → 403. M: plaintext → 400, rogue CA → verify failed, login роли `api` с cert `intruder`/`lb`/без cert → 400. | +| Оператор → OpenBao/Grafana/S3 admin | **Сделано.** OpenBao — без публикации порта: `docker compose exec openbao bao …` (TLS к 127.0.0.1:8200, root token только в env операторского процесса из `$OPERATOR_DIR/openbao/init.json`). Grafana — `https://localhost:3000` (public-ca), admin/пароль из Docker secret, sign-up и anonymous выключены, cookie secure. S3 admin-ключ — только job `s3-init` (из OpenBao), порт S3 не опубликован. | Отдельная операторская роль/policy вместо root — нет (root у оператора). | H: опубликованы только `127.0.0.1:8443` и `127.0.0.1:3000`. Healthcheck Grafana `curl --cacert public-ca https://localhost:3000/api/health`. | +| API → Redis/Valkey, если используется | **N/A** — Redis/Valkey в lab7 не используется: сессии и ключи идемпотентности хранятся в PostgreSQL, rate limit входа — в stick-table HAProxy. | — | compose.yaml: сервиса нет. | +| Replication/кластерные каналы, если добавлены | **N/A по составу** — репликации нет (один узел PostgreSQL, RabbitMQ, SeaweedFS, OpenBao Raft). Кластерные слушатели, которые продукты открывают сами, ограничены loopback: OpenBao `cluster_address 127.0.0.1:8201`; RabbitMQ epmd/Erlang dist на 127.0.0.1; SeaweedFS master/volume/filer `-ip.bind=127.0.0.1`; gRPC Loki/Tempo 127.0.0.1. | Модель отказа узла не заявляется (один Docker-host). | M: `internal-only port … from network` → refused. | Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. +### Scrape / health / bootstrap каналы + +| Канал | Как защищён | Проверка | +| --- | --- | --- | +| Docker healthcheck API/worker/relay/janitor | API: plaintext **только 127.0.0.1:8081** внутри контейнера, только `/health/live|ready` (остальное 404); worker/relay/janitor — файл heartbeat в tmpfs, без сети | M: `health listener 8081 from network` → refused | +| HAProxy → API readiness | тот же mTLS, что и трафик (`check` наследует `ssl verify required … crt lb.pem`) | M: identity `lb` → 200 | +| Healthchecks остальных | локально в контейнере: `bao status` (TLS 127.0.0.1), `su-exec postgres pg_isready` (сокет, peer), `curl --cacert … https://127.0.0.1:8333` (S3), `rabbitmq-diagnostics` (loopback dist), `promtool check ready --http.config.file` (mTLS), `curl --cacert public-ca https://localhost:3000` (Grafana) | compose.yaml | +| Prometheus scrape | только HTTPS + client cert `prometheus`, проверка CA и `server_name` цели | `up`=1 у всех целей; M — отказ без cert | +| Bootstrap PKI | `init-secrets.sh` локально, OpenSSL закреплённого образа, `--network none`; CA keys → `$OPERATOR_DIR` | H: `repo.scan_secrets_clean`, `image.no_key_or_secret_files` | +| Bootstrap OpenBao | `bao-init.sh` оператором: init/unseal/policies/roles/KV через `docker compose exec` (stdin, без вывода значений) | B: `status.*` | +| Bootstrap БД/S3 | job `migrate` (роль `postgres`, TLS verify-full, KV `messenger/migrate`, токен 5 мин), job `s3-init` (ключ admin, KV `messenger/s3-init`) — завершаются (Exited 0) | `docker compose ps -a` | + ## Обязательные отрицательные проверки -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. +1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». — **S + A (`tls.*`)**. +2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. — **M (`LB->API[0..1]`)**, для каждой реплики. +3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. — уровень 4 не заявлен; для broker и Collector выполнены проверки «без cert / чужой CA», identity-уровень — нет (M фиксирует `intruder` как принятый Collector-ом). +4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. — **A (`outsider.*`, `spoof.*`)**. +5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. — **B**. +6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. — уровень 4 не заявлен; подготовлено: на API и HAProxy отключены session tickets/кэш TLS-сессий (`OP_NO_TICKET`, `no-tls-tickets`, `tune.ssl.cachesize 0`, `no-ssl-reuse`), revoke-self токена OpenBao проверен (B). ## Bootstrap и эксплуатация В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. +**Решение:** CA keys и `openbao/init.json` (root token + 3 unseal shares, порог 2) — в `$OPERATOR_DIR` (700/600) у оператора, ни в один контейнер не монтируются. Leaf keys — в `secrets/pki` (каталог 700, вне Git и образа), каждый контейнер видит только свой ключ. После рестарта контейнера OpenBao он **sealed**: работающие API/worker продолжают обслуживать запросы (секреты уже в памяти), новые/перезапущенные процессы ждут (не становятся ready), оператор выполняет `./scripts/bao-unseal.sh` (2 из 3 shares через stdin). Обычный рестарт API unseal не требует. Подробно — [infra/openbao/README.md](../infra/openbao/README.md). + OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. +**Решение:** non-dev, integrated storage **Raft** (file storage в OpenBao 2.7 удалён) на volume `baodata`; **cert auth** (identity = тот же leaf-сертификат, что и для mTLS, без отдельного SecretID); токены service-типа с policy, TTL 15 мин / max 1 ч, renew-self и re-login; статические KV v2 secrets. + Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. diff --git a/evidence/README.md b/evidence/README.md index 8df7001..79e3306 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,24 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Lab7 — состав (2026-11-18, MacBook Air M3, Docker Desktop 28.0.4, VM 8 vCPU / 7.7 GiB) + +| Файл | Что | Команда | +| --- | --- | --- | +| run-all.txt | сводный прогон всех проверок | `WITH_LAB6=1 ./student-tests/run_all.sh` | +| clean-clone-run.txt | тот же прогон из чистой копии репозитория со свежими секретами/PKI/OpenBao | см. REPORT.md «Доказательства» | +| make-*.txt | `make check`, HTTPS smoke, внешний TLS trust | run_all | +| lab7-tls-matrix.json/.txt | 67 положительных/отрицательных TLS/mTLS проверок изнутри стенда | `checks lab7_tls_matrix.py` | +| lab7-api-security.json/.txt | 33 проверки внешнего API (TLS, ACL, logout/expiry, rate limit, лимиты, Argon2id) | `lab7_api_security.py` | +| lab7-openbao.json/.txt | 83 проверки OpenBao (статус, policies allow/deny, sealed-сценарий) | `lab7_openbao.py` | +| lab7-hardening.json/.txt | 182 проверки docker inspect / образа / репозитория / телеметрии | `lab7_hardening.py` | +| argon2-bench.json | замер профилей Argon2id в контейнере API | `argon2_bench.py` | +| test-persistence.txt, persistence-ps-*.txt | рестарт всех хранилищ (вкл. OpenBao + unseal), состояния до/после | `make test-persistence` | +| test-failover.txt | последовательная остановка api-1 и api-2 с alias-ами реплик | `make test-failover` | +| load*.json | нагрузка POST по HTTPS (до/после TCP_NODELAY) | `tests/load.py` | +| prometheus-up.txt | все scrape-цели `up=1` по HTTPS + client cert | obslib | +| obs-correlation.json, dashboard-check.json, worker-outage.txt | регрессии lab5/lab6 поверх TLS | run_all `WITH_LAB6=1` | + +Токены, пароли, ключи и unseal material в evidence не сохраняются (тесты печатают только статусы, +имена полей и SHA-256-префиксы). diff --git a/evidence/argon2-bench.json b/evidence/argon2-bench.json new file mode 100644 index 0000000..efad70e --- /dev/null +++ b/evidence/argon2-bench.json @@ -0,0 +1,120 @@ +{ + "measured_at": "2026-11-18T20:02:28Z", + "python": "3.12.7", + "machine": "aarch64", + "cgroup_cpu_limit": 0.5, + "cgroup_memory_limit_mib": 640, + "n_per_profile": 15, + "profiles": [ + { + "name": "owasp-min m=19MiB t=2 p=1 (выбран)", + "memory_kib": 19456, + "iterations": 2, + "parallelism": 1, + "hash_ms": { + "p50": 13.0, + "p95": 64.8, + "max": 65.7 + }, + "verify_ms": { + "p50": 13.2, + "p95": 63.2 + }, + "max_rss_growth_mib": 18.2, + "concurrency2_hashes_per_s": 27.2, + "sample_prefix": [ + "argon2id", + "v=19", + "m=19456,t=2,p=1" + ] + }, + { + "name": "owasp-alt m=46MiB t=1 p=1", + "memory_kib": 47104, + "iterations": 1, + "parallelism": 1, + "hash_ms": { + "p50": 21.6, + "p95": 71.3, + "max": 71.5 + }, + "verify_ms": { + "p50": 21.6, + "p95": 78.6 + }, + "max_rss_growth_mib": 46.0, + "concurrency2_hashes_per_s": 14.4, + "sample_prefix": [ + "argon2id", + "v=19", + "m=47104,t=1,p=1" + ] + }, + { + "name": "m=19MiB t=3 p=1", + "memory_kib": 19456, + "iterations": 3, + "parallelism": 1, + "hash_ms": { + "p50": 23.4, + "p95": 108.6, + "max": 183.1 + }, + "verify_ms": { + "p50": 69.7, + "p95": 105.5 + }, + "max_rss_growth_mib": 0.0, + "concurrency2_hashes_per_s": 16.6, + "sample_prefix": [ + "argon2id", + "v=19", + "m=19456,t=3,p=1" + ] + }, + { + "name": "m=64MiB t=3 p=1", + "memory_kib": 65536, + "iterations": 3, + "parallelism": 1, + "hash_ms": { + "p50": 186.4, + "p95": 207.8, + "max": 303.9 + }, + "verify_ms": { + "p50": 183.9, + "p95": 212.8 + }, + "max_rss_growth_mib": 0.0, + "concurrency2_hashes_per_s": 3.7, + "sample_prefix": [ + "argon2id", + "v=19", + "m=65536,t=3,p=1" + ] + }, + { + "name": "m=19MiB t=2 p=2", + "memory_kib": 19456, + "iterations": 2, + "parallelism": 2, + "hash_ms": { + "p50": 8.0, + "p95": 80.8, + "max": 83.2 + }, + "verify_ms": { + "p50": 15.9, + "p95": 92.4 + }, + "max_rss_growth_mib": 0.0, + "concurrency2_hashes_per_s": 15.0, + "sample_prefix": [ + "argon2id", + "v=19", + "m=19456,t=2,p=2" + ] + } + ] +} diff --git a/evidence/check-rules.txt b/evidence/check-rules.txt new file mode 100644 index 0000000..01937b1 --- /dev/null +++ b/evidence/check-rules.txt @@ -0,0 +1,19 @@ +== promtool check config (prom/prometheus:v3.13.5@sha256:aafa9d694e6a883c02db516d9f6e5b84e14195f49a2a6456a4cca815edda72b9) +Checking /etc/prometheus/prometheus.yml + SUCCESS: 1 rule files found + SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax + +Checking /etc/prometheus/rules/messenger.yml + SUCCESS: 26 rules found + +== promtool check web-config (TLS + обязательный client cert) +/etc/prometheus/web.yml SUCCESS +== promtool test rules + SUCCESS + +== otelcol validate (otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849) +collector config OK +== loki -verify-config (grafana/loki:3.6.19@sha256:0f9cbc340cbe33ee498e654075034336962057db072f57b37f253e14d476443a) +loki config OK +== dashboard JSON is up to date with scripts/build_dashboard.py +dashboard OK diff --git a/evidence/clean-clone-run.txt b/evidence/clean-clone-run.txt new file mode 100644 index 0000000..1a647b6 --- /dev/null +++ b/evidence/clean-clone-run.txt @@ -0,0 +1,88 @@ +# Прогон из чистой копии репозитория (без secrets/, .env, материала оператора), 2026-11-18 +# COMPOSE_PROJECT_NAME=mtusi-lab7-clean, отдельный OPERATOR_DIR; init-secrets -> check_rules -> build --no-cache -> bao-init -> up --wait -> WITH_LAB6=1 run_all.sh +created CA public-ca (key in operator dir) +created CA internal-ca (key in operator dir) +issued lb-public (public-ca, 30 days) +issued grafana-public (public-ca, 30 days) +issued lb (internal-ca, 30 days) +issued api (internal-ca, 30 days) +issued worker (internal-ca, 30 days) +issued relay (internal-ca, 30 days) +issued janitor (internal-ca, 30 days) +issued migrate (internal-ca, 30 days) +issued s3-init (internal-ca, 30 days) +issued ops (internal-ca, 30 days) +issued grafana (internal-ca, 30 days) +issued db (internal-ca, 30 days) +issued s3 (internal-ca, 30 days) +issued broker (internal-ca, 30 days) +issued otel-collector (internal-ca, 30 days) +issued prometheus (internal-ca, 30 days) +issued loki (internal-ca, 30 days) +issued tempo (internal-ca, 30 days) +issued openbao (internal-ca, 30 days) +issued intruder (internal-ca, 30 days) +issued rogue-lb (rogue-ca, 30 days) +== promtool check config (prom/prometheus:v3.13.5@sha256:aafa9d694e6a883c02db516d9f6e5b84e14195f49a2a6456a4cca815edda72b9) +== promtool check web-config (TLS + обязательный client cert) +== promtool test rules +== otelcol validate (otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849) +== loki -verify-config (grafana/loki:3.6.19@sha256:0f9cbc340cbe33ee498e654075034336962057db072f57b37f253e14d476443a) +== dashboard JSON is up to date with scripts/build_dashboard.py +openbao initialized; root token and 3 unseal shares saved to /clean/operator/openbao/init.json (operator only) +openbao unsealed (2 of 3 shares) +openbao configured: kv v2 'kv/', audit file->stdout, auth/cert roles: api worker relay janitor migrate s3-init ops + Container mtusi-lab7-clean-migrate-1 Exited + Container mtusi-lab7-clean-migrate-1 Exited + Container mtusi-lab7-clean-migrate-1 Exited + Container mtusi-lab7-clean-migrate-1 Exited + Container mtusi-lab7-clean-migrate-1 Exited + Container mtusi-lab7-clean-s3-init-1 Exited + Container mtusi-lab7-clean-s3-init-1 Exited + Container mtusi-lab7-clean-s3-init-1 Exited + Container mtusi-lab7-clean-migrate-1 Exited + Container mtusi-lab7-clean-s3-init-1 Exited +mtusi-lab7-clean-api-1 Up 11 seconds (healthy) +mtusi-lab7-clean-api-2 Up 11 seconds (healthy) +mtusi-lab7-clean-broker-1 Up 31 seconds (healthy) +mtusi-lab7-clean-db-1 Up 31 seconds (healthy) +mtusi-lab7-clean-grafana-1 Up 30 seconds (healthy) +mtusi-lab7-clean-janitor-1 Up 11 seconds (healthy) +mtusi-lab7-clean-lb-1 Up 10 seconds (healthy) +mtusi-lab7-clean-migrate-1 Exited (0) 27 seconds ago +mtusi-lab7-clean-openbao-1 Up 48 seconds (healthy) +mtusi-lab7-clean-prometheus-1 Up 31 seconds (healthy) +mtusi-lab7-clean-relay-1 Up 23 seconds (healthy) +mtusi-lab7-clean-s3-1 Up 31 seconds (healthy) +mtusi-lab7-clean-s3-init-1 Exited (0) 11 seconds ago +mtusi-lab7-clean-worker-1 Up 11 seconds (healthy) +mtusi-lab7-clean-worker-2 Up 11 seconds (healthy) +== make-check (2026-11-18T20:51:58Z) +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +== make-test (2026-11-18T20:51:58Z) +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +== make-test-security (2026-11-18T20:51:59Z) +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. +== lab7-tls-matrix (2026-11-18T20:51:59Z) +PASS: 67 checks, 0 failed +== lab7-api-security (2026-11-18T20:52:00Z) +PASS: 33 checks, 0 failed +== lab7-openbao (2026-11-18T20:52:36Z) +PASS: 83 checks, 0 failed +== lab7-hardening (2026-11-18T20:53:23Z) +PASS: 182 checks, 0 failed +== test-persistence (2026-11-18T20:53:44Z) +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +== test-failover (2026-11-18T20:54:33Z) +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== load (2026-11-18T20:54:56Z) +== scan-secrets (2026-11-18T20:55:26Z) +== check-rules (2026-11-18T20:55:26Z) +== obs-correlation (2026-11-18T20:55:29Z) +PASS: correlation request->trace->log and upload->relay->worker +== dashboard-check (2026-11-18T20:55:39Z) +== worker-outage (2026-11-18T20:56:20Z) +PASS: worker outage, api+broker restart, ready, duplicate delivery no-op +== done: ALL PASS ++(eval):1> echo 'CLEAN_EXIT=0' +CLEAN_EXIT=0 diff --git a/evidence/dashboard-check.json b/evidence/dashboard-check.json new file mode 100644 index 0000000..857a70c --- /dev/null +++ b/evidence/dashboard-check.json @@ -0,0 +1,598 @@ +{ + "dashboard": { + "uid": "messenger-overview", + "title": "Messenger — наблюдаемость (lab6)", + "provisioned": true, + "folder": "Messenger", + "panels": 56 + }, + "datasources": [ + "loki", + "prometheus", + "tempo" + ], + "datasource_health": { + "prometheus": "OK", + "loki": "OK", + "tempo": "OK" + }, + "panels": [ + { + "panel": "RPS бизнес-запросов", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "5xx, доля (2m)", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "p95 API (5m)", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "SLO доступности, 7d", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Остаток error budget, 7d", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Time to ready p95", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Ready lag очереди", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Alerts firing", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "История alerts (pending/firing -> resolved)", + "ref": "A", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "RPS по маршруту", + "ref": "A", + "datasource": "prometheus", + "series": 10, + "ok": true + }, + { + "panel": "Ответы по классу статуса", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Доля 5xx: API и LB", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Доля 5xx: API и LB", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Задержка API p50/p95/p99 (из histogram)", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Задержка API p50/p95/p99 (из histogram)", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Задержка API p50/p95/p99 (из histogram)", + "ref": "C", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "p95 по маршруту", + "ref": "A", + "datasource": "prometheus", + "series": 10, + "ok": true + }, + { + "panel": "5xx по причине (error.type)", + "ref": "A", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "Запросы в обработке (saturation API)", + "ref": "A", + "datasource": "prometheus", + "series": 6, + "ok": true + }, + { + "panel": "HAProxy: UP-реплики и очередь LB", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "HAProxy: UP-реплики и очередь LB", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Задания по исходу (в минуту)", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Длительность обработки job p50/p95/p99", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Длительность обработки job p50/p95/p99", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Длительность обработки job p50/p95/p99", + "ref": "C", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Time to ready p50/p95/p99", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Time to ready p50/p95/p99", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Time to ready p50/p95/p99", + "ref": "C", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Глубина очереди: PostgreSQL и RabbitMQ", + "ref": "A", + "datasource": "prometheus", + "series": 2, + "ok": true + }, + { + "panel": "Глубина очереди: PostgreSQL и RabbitMQ", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Глубина очереди: PostgreSQL и RabbitMQ", + "ref": "C", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Глубина очереди: PostgreSQL и RabbitMQ", + "ref": "D", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Глубина очереди: PostgreSQL и RabbitMQ", + "ref": "E", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Возраст старейшего задания и lag", + "ref": "A", + "datasource": "prometheus", + "series": 2, + "ok": true + }, + { + "panel": "Возраст старейшего задания и lag", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Возраст старейшего задания и lag", + "ref": "C", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Ожидание в очереди p95 (до claim)", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Ожидание в очереди p95 (до claim)", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Saturation worker: занятость потребителей", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Saturation worker: занятость потребителей", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Relay: публикации outbox", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Пул соединений API: used / idle / max", + "ref": "A", + "datasource": "prometheus", + "series": 12, + "ok": true + }, + { + "panel": "Пул соединений API: used / idle / max", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Ожидающие соединение и таймауты пула", + "ref": "A", + "datasource": "prometheus", + "series": 6, + "ok": true + }, + { + "panel": "Ожидающие соединение и таймауты пула", + "ref": "B", + "datasource": "prometheus", + "series": 2, + "ok": true + }, + { + "panel": "Ожидание соединения пула p95", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Операции БД p95 по операции (API)", + "ref": "A", + "datasource": "prometheus", + "series": 12, + "ok": true + }, + { + "panel": "Операции БД worker/relay p95", + "ref": "A", + "datasource": "prometheus", + "series": 5, + "ok": true + }, + { + "panel": "Ошибки операций БД", + "ref": "A", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "S3 p95 по операции", + "ref": "A", + "datasource": "prometheus", + "series": 4, + "ok": true + }, + { + "panel": "S3 ошибки", + "ref": "A", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "CPU, ядер (cgroup контейнера)", + "ref": "A", + "datasource": "prometheus", + "series": 6, + "ok": true + }, + { + "panel": "CPU saturation: доля лимита", + "ref": "A", + "datasource": "prometheus", + "series": 6, + "ok": true + }, + { + "panel": "Память контейнеров приложений", + "ref": "A", + "datasource": "prometheus", + "series": 14, + "ok": true + }, + { + "panel": "Память: доля лимита", + "ref": "A", + "datasource": "prometheus", + "series": 14, + "ok": true + }, + { + "panel": "Память telemetry-стека (RSS)", + "ref": "A", + "datasource": "prometheus", + "series": 3, + "ok": true + }, + { + "panel": "Память telemetry-стека (RSS)", + "ref": "B", + "datasource": "prometheus", + "series": 2, + "ok": true + }, + { + "panel": "Память telemetry-стека (RSS)", + "ref": "C", + "datasource": "prometheus", + "series": 2, + "ok": true + }, + { + "panel": "Память telemetry-стека (RSS)", + "ref": "D", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "CPU telemetry-стека", + "ref": "A", + "datasource": "prometheus", + "series": 3, + "ok": true + }, + { + "panel": "CPU telemetry-стека", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Collector: принято / отправлено / потеряно (spans)", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Collector: принято / отправлено / потеряно (spans)", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Collector: принято / отправлено / потеряно (spans)", + "ref": "C", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "Collector: принято / отправлено / потеряно (spans)", + "ref": "D", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "Collector: принято / отправлено / потеряно (spans)", + "ref": "E", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Collector: логи", + "ref": "A", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Collector: логи", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Collector: логи", + "ref": "C", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "Collector: логи", + "ref": "D", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "Collector: очередь экспортёров", + "ref": "A", + "datasource": "prometheus", + "series": 3, + "ok": true + }, + { + "panel": "Collector: очередь экспортёров", + "ref": "B", + "datasource": "prometheus", + "series": 3, + "ok": true + }, + { + "panel": "SDK приложений: потери на стороне процесса", + "ref": "A", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "SDK приложений: потери на стороне процесса", + "ref": "B", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "SDK приложений: потери на стороне процесса", + "ref": "C", + "datasource": "prometheus", + "series": 0, + "ok": true + }, + { + "panel": "SDK: заполнение очереди BatchSpanProcessor", + "ref": "A", + "datasource": "prometheus", + "series": 14, + "ok": true + }, + { + "panel": "SDK: заполнение очереди BatchSpanProcessor", + "ref": "B", + "datasource": "prometheus", + "series": 1, + "ok": true + }, + { + "panel": "Логи WARN/ERROR приложений (Loki)", + "ref": "A", + "datasource": "loki", + "lines": 20, + "ok": true + }, + { + "panel": "HTTP 5xx (event=http.request)", + "ref": "A", + "datasource": "loki", + "lines": 20, + "ok": true + }, + { + "panel": "Последние трассы обработки заданий (Tempo)", + "ref": "A", + "datasource": "tempo", + "traces": 5, + "ok": true + }, + { + "panel": "Медленные запросы API > 1 s (Tempo)", + "ref": "A", + "datasource": "tempo", + "traces": 0, + "ok": true + }, + { + "panel": "Трассы с ошибкой (Tempo)", + "ref": "A", + "datasource": "tempo", + "traces": 5, + "ok": true + } + ], + "query_errors": [], + "empty_required_panels": [], + "pass": true +} diff --git a/evidence/dashboard-check.txt b/evidence/dashboard-check.txt new file mode 100644 index 0000000..da3038a --- /dev/null +++ b/evidence/dashboard-check.txt @@ -0,0 +1,84 @@ +warmup: {'requests': 564, 'ok': 564, 'codes': {'200': 242, '201': 242, '202': 80}, 'error_ratio_5xx': 0.0, 'ok_rps': 14.1, 'p50_ms': 11.3, 'p95_ms': 26.6, 'p99_ms': 43.7} +ok RPS бизнес-запросов prometheus 1 +ok 5xx, доля (2m) prometheus 1 +ok p95 API (5m) prometheus 1 +ok SLO доступности, 7d prometheus 1 +ok Остаток error budget, 7d prometheus 1 +ok Time to ready p95 prometheus 1 +ok Ready lag очереди prometheus 1 +ok Alerts firing prometheus 1 +ok История alerts (pending/firing -> resolved) prometheus 0 +ok RPS по маршруту prometheus 10 +ok Ответы по классу статуса prometheus 1 +ok Доля 5xx: API и LB prometheus 1 +ok Доля 5xx: API и LB prometheus 1 +ok Задержка API p50/p95/p99 (из histogram) prometheus 1 +ok Задержка API p50/p95/p99 (из histogram) prometheus 1 +ok Задержка API p50/p95/p99 (из histogram) prometheus 1 +ok p95 по маршруту prometheus 10 +ok 5xx по причине (error.type) prometheus 0 +ok Запросы в обработке (saturation API) prometheus 6 +ok HAProxy: UP-реплики и очередь LB prometheus 1 +ok HAProxy: UP-реплики и очередь LB prometheus 1 +ok Задания по исходу (в минуту) prometheus 1 +ok Длительность обработки job p50/p95/p99 prometheus 1 +ok Длительность обработки job p50/p95/p99 prometheus 1 +ok Длительность обработки job p50/p95/p99 prometheus 1 +ok Time to ready p50/p95/p99 prometheus 1 +ok Time to ready p50/p95/p99 prometheus 1 +ok Time to ready p50/p95/p99 prometheus 1 +ok Глубина очереди: PostgreSQL и RabbitMQ prometheus 2 +ok Глубина очереди: PostgreSQL и RabbitMQ prometheus 1 +ok Глубина очереди: PostgreSQL и RabbitMQ prometheus 1 +ok Глубина очереди: PostgreSQL и RabbitMQ prometheus 1 +ok Глубина очереди: PostgreSQL и RabbitMQ prometheus 1 +ok Возраст старейшего задания и lag prometheus 2 +ok Возраст старейшего задания и lag prometheus 1 +ok Возраст старейшего задания и lag prometheus 1 +ok Ожидание в очереди p95 (до claim) prometheus 1 +ok Ожидание в очереди p95 (до claim) prometheus 1 +ok Saturation worker: занятость потребителей prometheus 1 +ok Saturation worker: занятость потребителей prometheus 1 +ok Relay: публикации outbox prometheus 1 +ok Пул соединений API: used / idle / max prometheus 12 +ok Пул соединений API: used / idle / max prometheus 1 +ok Ожидающие соединение и таймауты пула prometheus 6 +ok Ожидающие соединение и таймауты пула prometheus 2 +ok Ожидание соединения пула p95 prometheus 1 +ok Операции БД p95 по операции (API) prometheus 12 +ok Операции БД worker/relay p95 prometheus 5 +ok Ошибки операций БД prometheus 0 +ok S3 p95 по операции prometheus 4 +ok S3 ошибки prometheus 0 +ok CPU, ядер (cgroup контейнера) prometheus 6 +ok CPU saturation: доля лимита prometheus 6 +ok Память контейнеров приложений prometheus 14 +ok Память: доля лимита prometheus 14 +ok Память telemetry-стека (RSS) prometheus 3 +ok Память telemetry-стека (RSS) prometheus 2 +ok Память telemetry-стека (RSS) prometheus 2 +ok Память telemetry-стека (RSS) prometheus 1 +ok CPU telemetry-стека prometheus 3 +ok CPU telemetry-стека prometheus 1 +ok Collector: принято / отправлено / потеряно (spans) prometheus 1 +ok Collector: принято / отправлено / потеряно (spans) prometheus 1 +ok Collector: принято / отправлено / потеряно (spans) prometheus 0 +ok Collector: принято / отправлено / потеряно (spans) prometheus 0 +ok Collector: принято / отправлено / потеряно (spans) prometheus 1 +ok Collector: логи prometheus 1 +ok Collector: логи prometheus 1 +ok Collector: логи prometheus 0 +ok Collector: логи prometheus 0 +ok Collector: очередь экспортёров prometheus 3 +ok Collector: очередь экспортёров prometheus 3 +ok SDK приложений: потери на стороне процесса prometheus 0 +ok SDK приложений: потери на стороне процесса prometheus 0 +ok SDK приложений: потери на стороне процесса prometheus 0 +ok SDK: заполнение очереди BatchSpanProcessor prometheus 14 +ok SDK: заполнение очереди BatchSpanProcessor prometheus 1 +ok Логи WARN/ERROR приложений (Loki) loki 20 +ok HTTP 5xx (event=http.request) loki 20 +ok Последние трассы обработки заданий (Tempo) tempo 5 +ok Медленные запросы API > 1 s (Tempo) tempo 0 +ok Трассы с ошибкой (Tempo) tempo 5 +{"dashboard": {"uid": "messenger-overview", "title": "Messenger — наблюдаемость (lab6)", "provisioned": true, "folder": "Messenger", "panels": 56}, "datasource_health": {"prometheus": "OK", "loki": "OK", "tempo": "OK"}, "query_errors": [], "empty_required_panels": [], "pass": true} diff --git a/evidence/lab7-api-security.json b/evidence/lab7-api-security.json new file mode 100644 index 0000000..56e678a --- /dev/null +++ b/evidence/lab7-api-security.json @@ -0,0 +1,173 @@ +{ + "checked_at": "2026-11-18T20:27:20Z", + "base_url": "https://localhost:8443", + "total": 33, + "failed": 0, + "results": [ + { + "check": "tls.trusted_ca_and_hostname", + "pass": true, + "detail": "TLSv1.3" + }, + { + "check": "tls.wrong_hostname_rejected", + "pass": true, + "detail": "Hostname mismatch, certificate is not valid for 'evil.example'." + }, + { + "check": "tls.internal_ca_does_not_validate_ingress", + "pass": true, + "detail": "unable to get local issuer certificate" + }, + { + "check": "tls.tls11_refused", + "pass": true, + "detail": "tls_error TLSV1_ALERT_PROTOCOL_VERSION" + }, + { + "check": "setup.attachment_ready", + "pass": true, + "detail": "ready" + }, + { + "check": "outsider.chat_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.history_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.attachment_metadata_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.attachment_content_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.attachment_thumbnail_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.attach_foreign_file_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.post_into_foreign_chat_404", + "pass": true, + "detail": "404" + }, + { + "check": "outsider.chat_list_hides_foreign", + "pass": true, + "detail": "" + }, + { + "check": "spoof.x_user_id_ignored", + "pass": true, + "detail": "me=eve" + }, + { + "check": "spoof.x_user_id_alone_401", + "pass": true, + "detail": "" + }, + { + "check": "auth.basic_on_business_endpoint_401", + "pass": true, + "detail": "" + }, + { + "check": "auth.unknown_token_401", + "pass": true, + "detail": "Bearer realm=\"messenger\"" + }, + { + "check": "auth.same_401_unknown_user_and_wrong_password", + "pass": true, + "detail": "" + }, + { + "check": "auth.error_has_no_credentials_or_internals", + "pass": true, + "detail": "" + }, + { + "check": "logout.204", + "pass": true, + "detail": "" + }, + { + "check": "logout.token_revoked_401", + "pass": true, + "detail": "" + }, + { + "check": "logout.repeat_401", + "pass": true, + "detail": "" + }, + { + "check": "expiry.expired_session_401", + "pass": true, + "detail": "rows=1" + }, + { + "check": "argon2.stored_format", + "pass": true, + "detail": "$argon2id$v=19$m=19456,t=2,p=1$$" + }, + { + "check": "argon2.unique_salt", + "pass": true, + "detail": "" + }, + { + "check": "argon2.rehash_legacy_scrypt_on_login", + "pass": true, + "detail": "$argon2id$v=19$m=19456,t=2,p=1" + }, + { + "check": "limits.json_body_over_1mib_413", + "pass": true, + "detail": "payload_too_large" + }, + { + "check": "limits.upload_over_10mib_413", + "pass": true, + "detail": "payload_too_large" + }, + { + "check": "headers.instance_alias_random", + "pass": true, + "detail": "r-4676e5ef75" + }, + { + "check": "headers.no_server_banner", + "pass": true, + "detail": "None" + }, + { + "check": "headers.hsts", + "pass": true, + "detail": "" + }, + { + "check": "ratelimit.login_429_with_retry_after", + "pass": true, + "detail": "first 429 at attempt 19, Retry-After=10, code=rate_limited" + }, + { + "check": "ratelimit.recovers_after_window", + "pass": true, + "detail": "" + } + ] +} \ No newline at end of file diff --git a/evidence/lab7-api-security.txt b/evidence/lab7-api-security.txt new file mode 100644 index 0000000..4998e07 --- /dev/null +++ b/evidence/lab7-api-security.txt @@ -0,0 +1,34 @@ +PASS tls.trusted_ca_and_hostname TLSv1.3 +PASS tls.wrong_hostname_rejected Hostname mismatch, certificate is not valid for 'evil.example'. +PASS tls.internal_ca_does_not_validate_ingress unable to get local issuer certificate +PASS tls.tls11_refused tls_error TLSV1_ALERT_PROTOCOL_VERSION +PASS setup.attachment_ready ready +PASS outsider.chat_404 404 +PASS outsider.history_404 404 +PASS outsider.attachment_metadata_404 404 +PASS outsider.attachment_content_404 404 +PASS outsider.attachment_thumbnail_404 404 +PASS outsider.attach_foreign_file_404 404 +PASS outsider.post_into_foreign_chat_404 404 +PASS outsider.chat_list_hides_foreign +PASS spoof.x_user_id_ignored me=eve +PASS spoof.x_user_id_alone_401 +PASS auth.basic_on_business_endpoint_401 +PASS auth.unknown_token_401 Bearer realm="messenger" +PASS auth.same_401_unknown_user_and_wrong_password +PASS auth.error_has_no_credentials_or_internals +PASS logout.204 +PASS logout.token_revoked_401 +PASS logout.repeat_401 +PASS expiry.expired_session_401 rows=1 +PASS argon2.stored_format $argon2id$v=19$m=19456,t=2,p=1$$ +PASS argon2.unique_salt +PASS argon2.rehash_legacy_scrypt_on_login $argon2id$v=19$m=19456,t=2,p=1 +PASS limits.json_body_over_1mib_413 payload_too_large +PASS limits.upload_over_10mib_413 payload_too_large +PASS headers.instance_alias_random r-4676e5ef75 +PASS headers.no_server_banner None +PASS headers.hsts +PASS ratelimit.login_429_with_retry_after first 429 at attempt 19, Retry-After=10, code=rate_limited +PASS ratelimit.recovers_after_window +PASS: 33 checks, 0 failed diff --git a/evidence/lab7-hardening.json b/evidence/lab7-hardening.json new file mode 100644 index 0000000..3a4dea3 --- /dev/null +++ b/evidence/lab7-hardening.json @@ -0,0 +1,917 @@ +{ + "checked_at": "2026-11-18T20:59:25Z", + "total": 182, + "failed": 0, + "results": [ + { + "check": "api(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "api(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_apinet//private" + }, + { + "check": "api(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "api(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "api(1).memory_cpu_limits", + "pass": true, + "detail": "mem=640MiB cpu=0.50" + }, + { + "check": "api(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "api(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "api(1).non_root_uid", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "api(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "api(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "api(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "api(1).only_own_tls_identity_mounted", + "pass": true, + "detail": "/run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key" + }, + { + "check": "api(2).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "api(2).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_apinet//private" + }, + { + "check": "api(2).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "api(2).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "api(2).memory_cpu_limits", + "pass": true, + "detail": "mem=640MiB cpu=0.50" + }, + { + "check": "api(2).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "api(2).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "api(2).non_root_uid", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "api(2).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "api(2).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "api(2).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "api(2).only_own_tls_identity_mounted", + "pass": true, + "detail": "/run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key" + }, + { + "check": "broker(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "broker(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "broker(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "broker(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "broker(1).memory_cpu_limits", + "pass": true, + "detail": "mem=512MiB cpu=1.00" + }, + { + "check": "broker(1).cap_drop_all", + "pass": true, + "detail": "add=CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID" + }, + { + "check": "broker(1).cap_add_only_documented", + "pass": true, + "detail": "CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID" + }, + { + "check": "broker(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "broker(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "db(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "db(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "db(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "db(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "db(1).memory_cpu_limits", + "pass": true, + "detail": "mem=512MiB cpu=1.00" + }, + { + "check": "db(1).cap_drop_all", + "pass": true, + "detail": "add=CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID" + }, + { + "check": "db(1).cap_add_only_documented", + "pass": true, + "detail": "CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID" + }, + { + "check": "db(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "db(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_grafana-ui//private" + }, + { + "check": "grafana(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).memory_cpu_limits", + "pass": true, + "detail": "mem=384MiB cpu=0.50" + }, + { + "check": "grafana(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).published_3000/tcp", + "pass": true, + "detail": "127.0.0.1:3000" + }, + { + "check": "grafana(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "grafana(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "janitor(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).memory_cpu_limits", + "pass": true, + "detail": "mem=160MiB cpu=0.25" + }, + { + "check": "janitor(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).non_root_uid", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "janitor(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "janitor(1).only_own_tls_identity_mounted", + "pass": true, + "detail": "/run/secrets/service.key,/run/secrets/internal-ca.crt,/run/secrets/service.crt" + }, + { + "check": "lb(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_apinet//private" + }, + { + "check": "lb(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).memory_cpu_limits", + "pass": true, + "detail": "mem=128MiB cpu=0.50" + }, + { + "check": "lb(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).published_8443/tcp", + "pass": true, + "detail": "127.0.0.1:8443" + }, + { + "check": "lb(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "lb(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_telemetry//private" + }, + { + "check": "loki(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).memory_cpu_limits", + "pass": true, + "detail": "mem=512MiB cpu=0.50" + }, + { + "check": "loki(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "loki(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_vault//private" + }, + { + "check": "openbao(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).memory_cpu_limits", + "pass": true, + "detail": "mem=256MiB cpu=0.50" + }, + { + "check": "openbao(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "openbao(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_telemetry//private" + }, + { + "check": "otel-collector(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).memory_cpu_limits", + "pass": true, + "detail": "mem=256MiB cpu=0.50" + }, + { + "check": "otel-collector(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "otel-collector(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_telemetry//private" + }, + { + "check": "prometheus(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).memory_cpu_limits", + "pass": true, + "detail": "mem=512MiB cpu=0.50" + }, + { + "check": "prometheus(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "prometheus(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "relay(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).memory_cpu_limits", + "pass": true, + "detail": "mem=128MiB cpu=0.25" + }, + { + "check": "relay(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).non_root_uid", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "relay(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "relay(1).only_own_tls_identity_mounted", + "pass": true, + "detail": "/run/secrets/service.key,/run/secrets/internal-ca.crt,/run/secrets/service.crt" + }, + { + "check": "s3(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "s3(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "s3(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "s3(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "s3(1).memory_cpu_limits", + "pass": true, + "detail": "mem=768MiB cpu=1.00" + }, + { + "check": "s3(1).cap_drop_all", + "pass": true, + "detail": "add=CHOWN,SETGID,SETUID" + }, + { + "check": "s3(1).cap_add_only_documented", + "pass": true, + "detail": "CHOWN,SETGID,SETUID" + }, + { + "check": "s3(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "s3(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_telemetry//private" + }, + { + "check": "tempo(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).memory_cpu_limits", + "pass": true, + "detail": "mem=512MiB cpu=0.50" + }, + { + "check": "tempo(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "tempo(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "worker(1).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).memory_cpu_limits", + "pass": true, + "detail": "mem=1024MiB cpu=1.00" + }, + { + "check": "worker(1).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).non_root_uid", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "worker(1).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "worker(1).only_own_tls_identity_mounted", + "pass": true, + "detail": "/run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key" + }, + { + "check": "worker(2).not_privileged", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).no_host_namespaces", + "pass": true, + "detail": "mtusi-lab7_backend//private" + }, + { + "check": "worker(2).no_docker_socket", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).no_new_privileges", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).memory_cpu_limits", + "pass": true, + "detail": "mem=1024MiB cpu=1.00" + }, + { + "check": "worker(2).cap_drop_all", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).cap_add_only_documented", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).non_root_uid", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "worker(2).read_only_rootfs", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).no_vault_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).no_secret_values_in_env", + "pass": true, + "detail": "" + }, + { + "check": "worker(2).only_own_tls_identity_mounted", + "pass": true, + "detail": "/run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key" + }, + { + "check": "api(6506bf).health_listener_only_health", + "pass": true, + "detail": "{\"GET /health/ready\": \"200\", \"GET /api/v1/chats\": \"404\", \"POST /api/v1/users\": \"404\", \"GET /api/v1/users/me\": \"404\"}" + }, + { + "check": "api(c66490).health_listener_only_health", + "pass": true, + "detail": "{\"GET /health/ready\": \"200\", \"GET /api/v1/chats\": \"404\", \"POST /api/v1/users\": \"404\", \"GET /api/v1/users/me\": \"404\"}" + }, + { + "check": "image.no_key_or_secret_files", + "pass": true, + "detail": " no_secrets_dir " + }, + { + "check": "image.no_secrets_dir", + "pass": true, + "detail": "" + }, + { + "check": "image.history_without_secret_values", + "pass": true, + "detail": "" + }, + { + "check": "image.default_user_non_root", + "pass": true, + "detail": "10001:10001" + }, + { + "check": "repo.scan_secrets_clean", + "pass": true, + "detail": "checked 125 secret values from 50 files; files with leaks: none" + }, + { + "check": "telemetry.loki_search_positive_control", + "pass": true, + "detail": "matching lines: 2" + }, + { + "check": "telemetry.loki_no_bearer_token", + "pass": true, + "detail": "hits=0" + }, + { + "check": "telemetry.loki_no_password", + "pass": true, + "detail": "hits=0" + }, + { + "check": "telemetry.loki_no_token_prefix", + "pass": true, + "detail": "hits=0" + }, + { + "check": "telemetry.tempo_no_token_or_password", + "pass": true, + "detail": "checked 5 traces" + } + ] +} \ No newline at end of file diff --git a/evidence/lab7-hardening.txt b/evidence/lab7-hardening.txt new file mode 100644 index 0000000..bcee911 --- /dev/null +++ b/evidence/lab7-hardening.txt @@ -0,0 +1,183 @@ +PASS api(1).not_privileged +PASS api(1).no_host_namespaces mtusi-lab7_apinet//private +PASS api(1).no_docker_socket +PASS api(1).no_new_privileges +PASS api(1).memory_cpu_limits mem=640MiB cpu=0.50 +PASS api(1).cap_drop_all +PASS api(1).cap_add_only_documented +PASS api(1).non_root_uid 10001:10001 +PASS api(1).read_only_rootfs +PASS api(1).no_vault_token_in_env +PASS api(1).no_secret_values_in_env +PASS api(1).only_own_tls_identity_mounted /run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key +PASS api(2).not_privileged +PASS api(2).no_host_namespaces mtusi-lab7_apinet//private +PASS api(2).no_docker_socket +PASS api(2).no_new_privileges +PASS api(2).memory_cpu_limits mem=640MiB cpu=0.50 +PASS api(2).cap_drop_all +PASS api(2).cap_add_only_documented +PASS api(2).non_root_uid 10001:10001 +PASS api(2).read_only_rootfs +PASS api(2).no_vault_token_in_env +PASS api(2).no_secret_values_in_env +PASS api(2).only_own_tls_identity_mounted /run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key +PASS broker(1).not_privileged +PASS broker(1).no_host_namespaces mtusi-lab7_backend//private +PASS broker(1).no_docker_socket +PASS broker(1).no_new_privileges +PASS broker(1).memory_cpu_limits mem=512MiB cpu=1.00 +PASS broker(1).cap_drop_all add=CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID +PASS broker(1).cap_add_only_documented CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID +PASS broker(1).no_vault_token_in_env +PASS broker(1).no_secret_values_in_env +PASS db(1).not_privileged +PASS db(1).no_host_namespaces mtusi-lab7_backend//private +PASS db(1).no_docker_socket +PASS db(1).no_new_privileges +PASS db(1).memory_cpu_limits mem=512MiB cpu=1.00 +PASS db(1).cap_drop_all add=CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID +PASS db(1).cap_add_only_documented CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID +PASS db(1).no_vault_token_in_env +PASS db(1).no_secret_values_in_env +PASS grafana(1).not_privileged +PASS grafana(1).no_host_namespaces mtusi-lab7_grafana-ui//private +PASS grafana(1).no_docker_socket +PASS grafana(1).no_new_privileges +PASS grafana(1).memory_cpu_limits mem=384MiB cpu=0.50 +PASS grafana(1).cap_drop_all +PASS grafana(1).cap_add_only_documented +PASS grafana(1).published_3000/tcp 127.0.0.1:3000 +PASS grafana(1).no_vault_token_in_env +PASS grafana(1).no_secret_values_in_env +PASS janitor(1).not_privileged +PASS janitor(1).no_host_namespaces mtusi-lab7_backend//private +PASS janitor(1).no_docker_socket +PASS janitor(1).no_new_privileges +PASS janitor(1).memory_cpu_limits mem=160MiB cpu=0.25 +PASS janitor(1).cap_drop_all +PASS janitor(1).cap_add_only_documented +PASS janitor(1).non_root_uid 10001:10001 +PASS janitor(1).read_only_rootfs +PASS janitor(1).no_vault_token_in_env +PASS janitor(1).no_secret_values_in_env +PASS janitor(1).only_own_tls_identity_mounted /run/secrets/service.key,/run/secrets/internal-ca.crt,/run/secrets/service.crt +PASS lb(1).not_privileged +PASS lb(1).no_host_namespaces mtusi-lab7_apinet//private +PASS lb(1).no_docker_socket +PASS lb(1).no_new_privileges +PASS lb(1).memory_cpu_limits mem=128MiB cpu=0.50 +PASS lb(1).cap_drop_all +PASS lb(1).cap_add_only_documented +PASS lb(1).read_only_rootfs +PASS lb(1).published_8443/tcp 127.0.0.1:8443 +PASS lb(1).no_vault_token_in_env +PASS lb(1).no_secret_values_in_env +PASS loki(1).not_privileged +PASS loki(1).no_host_namespaces mtusi-lab7_telemetry//private +PASS loki(1).no_docker_socket +PASS loki(1).no_new_privileges +PASS loki(1).memory_cpu_limits mem=512MiB cpu=0.50 +PASS loki(1).cap_drop_all +PASS loki(1).cap_add_only_documented +PASS loki(1).read_only_rootfs +PASS loki(1).no_vault_token_in_env +PASS loki(1).no_secret_values_in_env +PASS openbao(1).not_privileged +PASS openbao(1).no_host_namespaces mtusi-lab7_vault//private +PASS openbao(1).no_docker_socket +PASS openbao(1).no_new_privileges +PASS openbao(1).memory_cpu_limits mem=256MiB cpu=0.50 +PASS openbao(1).cap_drop_all +PASS openbao(1).cap_add_only_documented +PASS openbao(1).read_only_rootfs +PASS openbao(1).no_vault_token_in_env +PASS openbao(1).no_secret_values_in_env +PASS otel-collector(1).not_privileged +PASS otel-collector(1).no_host_namespaces mtusi-lab7_telemetry//private +PASS otel-collector(1).no_docker_socket +PASS otel-collector(1).no_new_privileges +PASS otel-collector(1).memory_cpu_limits mem=256MiB cpu=0.50 +PASS otel-collector(1).cap_drop_all +PASS otel-collector(1).cap_add_only_documented +PASS otel-collector(1).read_only_rootfs +PASS otel-collector(1).no_vault_token_in_env +PASS otel-collector(1).no_secret_values_in_env +PASS prometheus(1).not_privileged +PASS prometheus(1).no_host_namespaces mtusi-lab7_telemetry//private +PASS prometheus(1).no_docker_socket +PASS prometheus(1).no_new_privileges +PASS prometheus(1).memory_cpu_limits mem=512MiB cpu=0.50 +PASS prometheus(1).cap_drop_all +PASS prometheus(1).cap_add_only_documented +PASS prometheus(1).read_only_rootfs +PASS prometheus(1).no_vault_token_in_env +PASS prometheus(1).no_secret_values_in_env +PASS relay(1).not_privileged +PASS relay(1).no_host_namespaces mtusi-lab7_backend//private +PASS relay(1).no_docker_socket +PASS relay(1).no_new_privileges +PASS relay(1).memory_cpu_limits mem=128MiB cpu=0.25 +PASS relay(1).cap_drop_all +PASS relay(1).cap_add_only_documented +PASS relay(1).non_root_uid 10001:10001 +PASS relay(1).read_only_rootfs +PASS relay(1).no_vault_token_in_env +PASS relay(1).no_secret_values_in_env +PASS relay(1).only_own_tls_identity_mounted /run/secrets/service.key,/run/secrets/internal-ca.crt,/run/secrets/service.crt +PASS s3(1).not_privileged +PASS s3(1).no_host_namespaces mtusi-lab7_backend//private +PASS s3(1).no_docker_socket +PASS s3(1).no_new_privileges +PASS s3(1).memory_cpu_limits mem=768MiB cpu=1.00 +PASS s3(1).cap_drop_all add=CHOWN,SETGID,SETUID +PASS s3(1).cap_add_only_documented CHOWN,SETGID,SETUID +PASS s3(1).no_vault_token_in_env +PASS s3(1).no_secret_values_in_env +PASS tempo(1).not_privileged +PASS tempo(1).no_host_namespaces mtusi-lab7_telemetry//private +PASS tempo(1).no_docker_socket +PASS tempo(1).no_new_privileges +PASS tempo(1).memory_cpu_limits mem=512MiB cpu=0.50 +PASS tempo(1).cap_drop_all +PASS tempo(1).cap_add_only_documented +PASS tempo(1).read_only_rootfs +PASS tempo(1).no_vault_token_in_env +PASS tempo(1).no_secret_values_in_env +PASS worker(1).not_privileged +PASS worker(1).no_host_namespaces mtusi-lab7_backend//private +PASS worker(1).no_docker_socket +PASS worker(1).no_new_privileges +PASS worker(1).memory_cpu_limits mem=1024MiB cpu=1.00 +PASS worker(1).cap_drop_all +PASS worker(1).cap_add_only_documented +PASS worker(1).non_root_uid 10001:10001 +PASS worker(1).read_only_rootfs +PASS worker(1).no_vault_token_in_env +PASS worker(1).no_secret_values_in_env +PASS worker(1).only_own_tls_identity_mounted /run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key +PASS worker(2).not_privileged +PASS worker(2).no_host_namespaces mtusi-lab7_backend//private +PASS worker(2).no_docker_socket +PASS worker(2).no_new_privileges +PASS worker(2).memory_cpu_limits mem=1024MiB cpu=1.00 +PASS worker(2).cap_drop_all +PASS worker(2).cap_add_only_documented +PASS worker(2).non_root_uid 10001:10001 +PASS worker(2).read_only_rootfs +PASS worker(2).no_vault_token_in_env +PASS worker(2).no_secret_values_in_env +PASS worker(2).only_own_tls_identity_mounted /run/secrets/internal-ca.crt,/run/secrets/service.crt,/run/secrets/service.key +PASS api(6506bf).health_listener_only_health {"GET /health/ready": "200", "GET /api/v1/chats": "404", "POST /api/v1/users": "404", "GET /api/v1/users/me": "404"} +PASS api(c66490).health_listener_only_health {"GET /health/ready": "200", "GET /api/v1/chats": "404", "POST /api/v1/users": "404", "GET /api/v1/users/me": "404"} +PASS image.no_key_or_secret_files no_secrets_dir +PASS image.no_secrets_dir +PASS image.history_without_secret_values +PASS image.default_user_non_root 10001:10001 +PASS repo.scan_secrets_clean checked 125 secret values from 50 files; files with leaks: none +PASS telemetry.loki_search_positive_control matching lines: 2 +PASS telemetry.loki_no_bearer_token hits=0 +PASS telemetry.loki_no_password hits=0 +PASS telemetry.loki_no_token_prefix hits=0 +PASS telemetry.tempo_no_token_or_password checked 5 traces +PASS: 182 checks, 0 failed diff --git a/evidence/lab7-openbao.json b/evidence/lab7-openbao.json new file mode 100644 index 0000000..945574c --- /dev/null +++ b/evidence/lab7-openbao.json @@ -0,0 +1,422 @@ +{ + "checked_at": "2026-11-18T20:28:13Z", + "total": 83, + "failed": 0, + "results": [ + { + "check": "status.initialized_unsealed", + "pass": true, + "detail": "" + }, + { + "check": "status.storage_raft_persistent", + "pass": true, + "detail": "raft" + }, + { + "check": "status.shares_threshold", + "pass": true, + "detail": "t=2 n=3" + }, + { + "check": "status.not_dev_mode", + "pass": true, + "detail": "[\"bao\",\"server\",\"-config=/openbao/config/openbao.hcl\"] null" + }, + { + "check": "api.login_policies", + "pass": true, + "detail": "default,messenger-api" + }, + { + "check": "api.token_ttl_bounded", + "pass": true, + "detail": "ttl=900" + }, + { + "check": "api.read_own_path_200", + "pass": true, + "detail": "" + }, + { + "check": "api.own_fields_exact", + "pass": true, + "detail": "POSTGRES_PASSWORD,S3_ACCESS_KEY_ID,S3_SECRET_ACCESS_KEY" + }, + { + "check": "api.deny_read_other_service", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_read_migrate_admin", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_list_metadata", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_write_own_secret", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_write_policy", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_create_token", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_seal", + "pass": true, + "detail": "403" + }, + { + "check": "api.deny_list_auth_roles", + "pass": true, + "detail": "403" + }, + { + "check": "api.renew_self", + "pass": true, + "detail": "" + }, + { + "check": "api.revoked_token_denied", + "pass": true, + "detail": "403" + }, + { + "check": "api.no_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "api.only_tls_identity_mounted", + "pass": true, + "detail": "internal-ca.crt,service.crt,service.key" + }, + { + "check": "api.db_role_not_privileged", + "pass": true, + "detail": "api_svc TLSv1.3" + }, + { + "check": "worker.login_policies", + "pass": true, + "detail": "default,messenger-worker" + }, + { + "check": "worker.token_ttl_bounded", + "pass": true, + "detail": "ttl=900" + }, + { + "check": "worker.read_own_path_200", + "pass": true, + "detail": "" + }, + { + "check": "worker.own_fields_exact", + "pass": true, + "detail": "BROKER_URL,POSTGRES_PASSWORD,S3_ACCESS_KEY_ID,S3_SECRET_ACCESS_KEY" + }, + { + "check": "worker.deny_read_other_service", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_read_migrate_admin", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_list_metadata", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_write_own_secret", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_write_policy", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_create_token", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_seal", + "pass": true, + "detail": "403" + }, + { + "check": "worker.deny_list_auth_roles", + "pass": true, + "detail": "403" + }, + { + "check": "worker.renew_self", + "pass": true, + "detail": "" + }, + { + "check": "worker.revoked_token_denied", + "pass": true, + "detail": "403" + }, + { + "check": "worker.no_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "worker.only_tls_identity_mounted", + "pass": true, + "detail": "internal-ca.crt,service.crt,service.key" + }, + { + "check": "worker.db_role_not_privileged", + "pass": true, + "detail": "worker_svc TLSv1.3" + }, + { + "check": "relay.login_policies", + "pass": true, + "detail": "default,messenger-relay" + }, + { + "check": "relay.token_ttl_bounded", + "pass": true, + "detail": "ttl=900" + }, + { + "check": "relay.read_own_path_200", + "pass": true, + "detail": "" + }, + { + "check": "relay.own_fields_exact", + "pass": true, + "detail": "BROKER_URL,POSTGRES_PASSWORD" + }, + { + "check": "relay.deny_read_other_service", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_read_migrate_admin", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_list_metadata", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_write_own_secret", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_write_policy", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_create_token", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_seal", + "pass": true, + "detail": "403" + }, + { + "check": "relay.deny_list_auth_roles", + "pass": true, + "detail": "403" + }, + { + "check": "relay.renew_self", + "pass": true, + "detail": "" + }, + { + "check": "relay.revoked_token_denied", + "pass": true, + "detail": "403" + }, + { + "check": "relay.no_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "relay.only_tls_identity_mounted", + "pass": true, + "detail": "internal-ca.crt,service.crt,service.key" + }, + { + "check": "relay.db_role_not_privileged", + "pass": true, + "detail": "relay_svc TLSv1.3" + }, + { + "check": "janitor.login_policies", + "pass": true, + "detail": "default,messenger-janitor" + }, + { + "check": "janitor.token_ttl_bounded", + "pass": true, + "detail": "ttl=900" + }, + { + "check": "janitor.read_own_path_200", + "pass": true, + "detail": "" + }, + { + "check": "janitor.own_fields_exact", + "pass": true, + "detail": "POSTGRES_PASSWORD,S3_ACCESS_KEY_ID,S3_SECRET_ACCESS_KEY" + }, + { + "check": "janitor.deny_read_other_service", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_read_migrate_admin", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_list_metadata", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_write_own_secret", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_write_policy", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_create_token", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_seal", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.deny_list_auth_roles", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.renew_self", + "pass": true, + "detail": "" + }, + { + "check": "janitor.revoked_token_denied", + "pass": true, + "detail": "403" + }, + { + "check": "janitor.no_token_in_env", + "pass": true, + "detail": "" + }, + { + "check": "janitor.only_tls_identity_mounted", + "pass": true, + "detail": "internal-ca.crt,service.crt,service.key" + }, + { + "check": "janitor.db_role_not_privileged", + "pass": true, + "detail": "janitor_svc TLSv1.3" + }, + { + "check": "worker.has_own_s3_key", + "pass": true, + "detail": "" + }, + { + "check": "worker.no_admin_or_api_s3_key", + "pass": true, + "detail": "" + }, + { + "check": "worker.db_no_users_sessions_messages", + "pass": true, + "detail": "" + }, + { + "check": "api.db_no_ddl_truncate", + "pass": true, + "detail": "" + }, + { + "check": "sealed.after_restart_sealed", + "pass": true, + "detail": "" + }, + { + "check": "sealed.running_api_keeps_serving", + "pass": true, + "detail": "" + }, + { + "check": "sealed.restarted_replica_not_healthy", + "pass": true, + "detail": "unhealthy" + }, + { + "check": "sealed.restarted_replica_waits_for_openbao", + "pass": true, + "detail": "" + }, + { + "check": "sealed.lb_serves_from_other_replica", + "pass": true, + "detail": "r-752aa9a6c1" + }, + { + "check": "sealed.replica_healthy_after_unseal", + "pass": true, + "detail": "11.5s after unseal (total 42s)" + }, + { + "check": "sealed.kv_persisted_after_restart", + "pass": true, + "detail": "" + } + ] +} \ No newline at end of file diff --git a/evidence/lab7-openbao.txt b/evidence/lab7-openbao.txt new file mode 100644 index 0000000..f3ebbae --- /dev/null +++ b/evidence/lab7-openbao.txt @@ -0,0 +1,84 @@ +PASS status.initialized_unsealed +PASS status.storage_raft_persistent raft +PASS status.shares_threshold t=2 n=3 +PASS status.not_dev_mode ["bao","server","-config=/openbao/config/openbao.hcl"] null +PASS api.login_policies default,messenger-api +PASS api.token_ttl_bounded ttl=900 +PASS api.read_own_path_200 +PASS api.own_fields_exact POSTGRES_PASSWORD,S3_ACCESS_KEY_ID,S3_SECRET_ACCESS_KEY +PASS api.deny_read_other_service 403 +PASS api.deny_read_migrate_admin 403 +PASS api.deny_list_metadata 403 +PASS api.deny_write_own_secret 403 +PASS api.deny_write_policy 403 +PASS api.deny_create_token 403 +PASS api.deny_seal 403 +PASS api.deny_list_auth_roles 403 +PASS api.renew_self +PASS api.revoked_token_denied 403 +PASS api.no_token_in_env +PASS api.only_tls_identity_mounted internal-ca.crt,service.crt,service.key +PASS api.db_role_not_privileged api_svc TLSv1.3 +PASS worker.login_policies default,messenger-worker +PASS worker.token_ttl_bounded ttl=900 +PASS worker.read_own_path_200 +PASS worker.own_fields_exact BROKER_URL,POSTGRES_PASSWORD,S3_ACCESS_KEY_ID,S3_SECRET_ACCESS_KEY +PASS worker.deny_read_other_service 403 +PASS worker.deny_read_migrate_admin 403 +PASS worker.deny_list_metadata 403 +PASS worker.deny_write_own_secret 403 +PASS worker.deny_write_policy 403 +PASS worker.deny_create_token 403 +PASS worker.deny_seal 403 +PASS worker.deny_list_auth_roles 403 +PASS worker.renew_self +PASS worker.revoked_token_denied 403 +PASS worker.no_token_in_env +PASS worker.only_tls_identity_mounted internal-ca.crt,service.crt,service.key +PASS worker.db_role_not_privileged worker_svc TLSv1.3 +PASS relay.login_policies default,messenger-relay +PASS relay.token_ttl_bounded ttl=900 +PASS relay.read_own_path_200 +PASS relay.own_fields_exact BROKER_URL,POSTGRES_PASSWORD +PASS relay.deny_read_other_service 403 +PASS relay.deny_read_migrate_admin 403 +PASS relay.deny_list_metadata 403 +PASS relay.deny_write_own_secret 403 +PASS relay.deny_write_policy 403 +PASS relay.deny_create_token 403 +PASS relay.deny_seal 403 +PASS relay.deny_list_auth_roles 403 +PASS relay.renew_self +PASS relay.revoked_token_denied 403 +PASS relay.no_token_in_env +PASS relay.only_tls_identity_mounted internal-ca.crt,service.crt,service.key +PASS relay.db_role_not_privileged relay_svc TLSv1.3 +PASS janitor.login_policies default,messenger-janitor +PASS janitor.token_ttl_bounded ttl=900 +PASS janitor.read_own_path_200 +PASS janitor.own_fields_exact POSTGRES_PASSWORD,S3_ACCESS_KEY_ID,S3_SECRET_ACCESS_KEY +PASS janitor.deny_read_other_service 403 +PASS janitor.deny_read_migrate_admin 403 +PASS janitor.deny_list_metadata 403 +PASS janitor.deny_write_own_secret 403 +PASS janitor.deny_write_policy 403 +PASS janitor.deny_create_token 403 +PASS janitor.deny_seal 403 +PASS janitor.deny_list_auth_roles 403 +PASS janitor.renew_self +PASS janitor.revoked_token_denied 403 +PASS janitor.no_token_in_env +PASS janitor.only_tls_identity_mounted internal-ca.crt,service.crt,service.key +PASS janitor.db_role_not_privileged janitor_svc TLSv1.3 +PASS worker.has_own_s3_key +PASS worker.no_admin_or_api_s3_key +PASS worker.db_no_users_sessions_messages +PASS api.db_no_ddl_truncate +PASS sealed.after_restart_sealed +PASS sealed.running_api_keeps_serving +PASS sealed.restarted_replica_not_healthy unhealthy +PASS sealed.restarted_replica_waits_for_openbao +PASS sealed.lb_serves_from_other_replica r-752aa9a6c1 +PASS sealed.replica_healthy_after_unseal 11.5s after unseal (total 42s) +PASS sealed.kv_persisted_after_restart +PASS: 83 checks, 0 failed diff --git a/evidence/lab7-tls-matrix.json b/evidence/lab7-tls-matrix.json new file mode 100644 index 0000000..d8251b4 --- /dev/null +++ b/evidence/lab7-tls-matrix.json @@ -0,0 +1,607 @@ +{ + "checked_at": "2026-11-18T20:26:44Z", + "total": 67, + "failed": 0, + "results": [ + { + "link": "LB->API", + "case": "replicas_resolved", + "pass": true, + "detail": 2 + }, + { + "link": "LB->API[0]", + "case": "no client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "LB->API[0]", + "case": "client cert of rogue CA, SAN lb", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "LB->API[0]", + "case": "trusted CA, identity intruder", + "expect": "fail", + "observed": "fail", + "detail": "closed_after_handshake_no_http_response", + "pass": true, + "note": "" + }, + { + "link": "LB->API[0]", + "case": "plaintext HTTP to 8443", + "expect": "fail", + "observed": "fail", + "detail": "reset_by_server", + "pass": true, + "note": "" + }, + { + "link": "LB->API[0]", + "case": "health listener 8081 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "LB->API[0]", + "case": "server cert, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'db'.", + "pass": true, + "note": "" + }, + { + "link": "LB->API[0]", + "case": "identity lb -> /health/ready", + "expect": "ok", + "observed": "ok", + "detail": 200, + "pass": true, + "note": "status=200" + }, + { + "link": "LB->API[0]", + "case": "identity lb, no Bearer -> business API", + "expect": "ok", + "observed": "ok", + "detail": 401, + "pass": true, + "note": "mTLS не заменяет сессию: ожидается 401, получено 401" + }, + { + "link": "LB->API[1]", + "case": "no client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "LB->API[1]", + "case": "client cert of rogue CA, SAN lb", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "LB->API[1]", + "case": "trusted CA, identity intruder", + "expect": "fail", + "observed": "fail", + "detail": "closed_after_handshake_no_http_response", + "pass": true, + "note": "" + }, + { + "link": "LB->API[1]", + "case": "plaintext HTTP to 8443", + "expect": "fail", + "observed": "fail", + "detail": "reset_by_server", + "pass": true, + "note": "" + }, + { + "link": "LB->API[1]", + "case": "health listener 8081 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "LB->API[1]", + "case": "server cert, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'db'.", + "pass": true, + "note": "" + }, + { + "link": "LB->API[1]", + "case": "identity lb -> /health/ready", + "expect": "ok", + "observed": "ok", + "detail": 200, + "pass": true, + "note": "status=200" + }, + { + "link": "LB->API[1]", + "case": "identity lb, no Bearer -> business API", + "expect": "ok", + "observed": "ok", + "detail": 401, + "pass": true, + "note": "mTLS не заменяет сессию: ожидается 401, получено 401" + }, + { + "link": "API/worker->PostgreSQL", + "case": "sslmode=disable (plaintext)", + "expect": "fail", + "observed": "fail", + "detail": "connection failed: connection to server at \"192.168.0.3\", port 5432 failed: fatal: pg_hba.conf rejects connection for h", + "pass": true, + "note": "" + }, + { + "link": "API/worker->PostgreSQL", + "case": "verify-full, rogue CA as trust", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client", + "pass": true, + "note": "" + }, + { + "link": "API/worker->PostgreSQL", + "case": "verify-full by IP (SAN mismatch)", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client", + "pass": true, + "note": "" + }, + { + "link": "API/worker->PostgreSQL", + "case": "verify-full, internal CA, wrong pwd", + "expect": "ok", + "observed": "ok", + "detail": "tls_ok_then_password_rejected", + "pass": true, + "note": "TLS проверен; дальше SCRAM отклоняет неверный пароль" + }, + { + "link": "API/worker->S3", + "case": "plaintext HTTP", + "expect": "fail", + "observed": "fail", + "detail": "http_400_plain_http_sent_to_https_port", + "pass": true, + "note": "" + }, + { + "link": "API/worker->S3", + "case": "HTTPS, rogue CA as trust", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:unable to get local issuer certificate", + "pass": true, + "note": "" + }, + { + "link": "API/worker->S3", + "case": "HTTPS verified, anonymous list", + "expect": "ok", + "observed": "ok", + "detail": 403, + "pass": true, + "note": "TLS ok; анонимно: 403 (ожидается 403)" + }, + { + "link": "API/worker->broker", + "case": "plaintext AMQP 5672", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "API/worker->broker", + "case": "AMQPS, rogue CA as trust", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:self-signed certificate in certificate chain", + "pass": true, + "note": "" + }, + { + "link": "API/worker->broker", + "case": "AMQPS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "verify_peer: предъявленный чужой сертификат отвергается" + }, + { + "link": "API/worker->broker", + "case": "AMQPS verified (без cert), Connection.Start", + "expect": "ok", + "observed": "ok", + "detail": "amqp_connection_start", + "pass": true, + "note": "далее аутентификация паролем + ACL vhost (проверка прав — broker_permissions)" + }, + { + "link": "API/worker->broker", + "case": "AMQPS, user worker, wrong password", + "expect": "fail", + "observed": "fail", + "detail": "auth_refused", + "pass": true, + "note": "" + }, + { + "link": "Prometheus->broker metrics", + "case": "HTTPS 15691 without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "Prometheus->broker metrics", + "case": "plaintext 15692 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "API/worker->Collector", + "case": "plaintext", + "expect": "fail", + "observed": "fail", + "detail": "http_400_plain_http_sent_to_https_port", + "pass": true, + "note": "" + }, + { + "link": "API/worker->Collector", + "case": "TLS without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "API/worker->Collector", + "case": "TLS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "API/worker->Collector", + "case": "TLS, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'.", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Prometheus", + "case": "plaintext", + "expect": "fail", + "observed": "fail", + "detail": "http_400_plain_http_sent_to_https_port", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Prometheus", + "case": "TLS without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Prometheus", + "case": "TLS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Prometheus", + "case": "TLS, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'.", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Loki", + "case": "plaintext", + "expect": "fail", + "observed": "fail", + "detail": "http_400_plain_http_sent_to_https_port", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Loki", + "case": "TLS without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Loki", + "case": "TLS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "Collector/Grafana->Loki", + "case": "TLS, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'.", + "pass": true, + "note": "" + }, + { + "link": "Grafana->Tempo", + "case": "plaintext", + "expect": "fail", + "observed": "fail", + "detail": "http_400_plain_http_sent_to_https_port", + "pass": true, + "note": "" + }, + { + "link": "Grafana->Tempo", + "case": "TLS without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "Grafana->Tempo", + "case": "TLS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:SSLEOFError", + "pass": true, + "note": "" + }, + { + "link": "Grafana->Tempo", + "case": "TLS, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'.", + "pass": true, + "note": "" + }, + { + "link": "Collector->Tempo OTLP gRPC", + "case": "plaintext", + "expect": "fail", + "observed": "fail", + "detail": "closed_without_plaintext_response", + "pass": true, + "note": "" + }, + { + "link": "Collector->Tempo OTLP gRPC", + "case": "TLS without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "Collector->Tempo OTLP gRPC", + "case": "TLS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "Collector->Tempo OTLP gRPC", + "case": "TLS, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'.", + "pass": true, + "note": "" + }, + { + "link": "Prometheus->LB metrics", + "case": "plaintext", + "expect": "fail", + "observed": "fail", + "detail": "closed_without_plaintext_response", + "pass": true, + "note": "" + }, + { + "link": "Prometheus->LB metrics", + "case": "TLS without client cert", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true, + "note": "" + }, + { + "link": "Prometheus->LB metrics", + "case": "TLS, client cert of rogue CA", + "expect": "fail", + "observed": "fail", + "detail": "tls_alert:TLSV1_ALERT_UNKNOWN_CA", + "pass": true, + "note": "" + }, + { + "link": "Prometheus->LB metrics", + "case": "TLS, wrong expected hostname", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'.", + "pass": true, + "note": "" + }, + { + "link": "internal-only port", + "case": "otel-collector:4317 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "internal-only port", + "case": "otel-collector:8888 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "internal-only port", + "case": "otel-collector:13133 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "internal-only port", + "case": "loki:9095 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "internal-only port", + "case": "tempo:9096 from network", + "expect": "fail", + "observed": "fail", + "detail": "connection_refused", + "pass": true, + "note": "" + }, + { + "link": "API/worker->Collector", + "case": "trusted CA, foreign identity (intruder)", + "expect": "documented", + "observed": "ok", + "detail": 405, + "pass": true, + "note": "level 3: CA-only mTLS; разграничение identity — уровень 4 (см. trust-matrix)" + }, + { + "link": "services->OpenBao", + "case": "plaintext HTTP", + "expect": "fail", + "observed": "fail", + "detail": "http_400_plain_http_sent_to_https_port", + "pass": true, + "note": "" + }, + { + "link": "services->OpenBao", + "case": "TLS, rogue CA as trust", + "expect": "fail", + "observed": "fail", + "detail": "server_cert_rejected_by_client:unable to get local issuer certificate", + "pass": true, + "note": "" + }, + { + "link": "services->OpenBao", + "case": "TLS verified, /v1/sys/health", + "expect": "ok", + "observed": "ok", + "detail": 200, + "pass": true, + "note": "" + }, + { + "link": "services->OpenBao", + "case": "cert login as role api with intruder cert", + "expect": "fail", + "observed": "fail", + "detail": "http_400", + "pass": true, + "note": "" + }, + { + "link": "services->OpenBao", + "case": "cert login as role api with lb cert", + "expect": "fail", + "observed": "fail", + "detail": "http_400", + "pass": true, + "note": "сертификат того же CA, но SAN не api -> роль не подходит" + }, + { + "link": "services->OpenBao", + "case": "cert login without client cert", + "expect": "fail", + "observed": "fail", + "detail": "http_400", + "pass": true, + "note": "" + } + ] +} \ No newline at end of file diff --git a/evidence/lab7-tls-matrix.txt b/evidence/lab7-tls-matrix.txt new file mode 100644 index 0000000..e58a5ec --- /dev/null +++ b/evidence/lab7-tls-matrix.txt @@ -0,0 +1,68 @@ +API replicas: 2 +PASS LB->API[0] no client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS LB->API[0] client cert of rogue CA, SAN lb expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS LB->API[0] trusted CA, identity intruder expect=fail got=fail closed_after_handshake_no_http_response +PASS LB->API[0] plaintext HTTP to 8443 expect=fail got=fail reset_by_server +PASS LB->API[0] health listener 8081 from network expect=fail got=fail connection_refused +PASS LB->API[0] server cert, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'db'. +PASS LB->API[0] identity lb -> /health/ready expect=ok got=ok 200 +PASS LB->API[0] identity lb, no Bearer -> business API expect=ok got=ok 401 +PASS LB->API[1] no client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS LB->API[1] client cert of rogue CA, SAN lb expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS LB->API[1] trusted CA, identity intruder expect=fail got=fail closed_after_handshake_no_http_response +PASS LB->API[1] plaintext HTTP to 8443 expect=fail got=fail reset_by_server +PASS LB->API[1] health listener 8081 from network expect=fail got=fail connection_refused +PASS LB->API[1] server cert, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'db'. +PASS LB->API[1] identity lb -> /health/ready expect=ok got=ok 200 +PASS LB->API[1] identity lb, no Bearer -> business API expect=ok got=ok 401 +PASS API/worker->PostgreSQL sslmode=disable (plaintext) expect=fail got=fail connection failed: connection to server at "192.168.0.3", port 5432 failed: fatal: pg_hba.conf rejects connection for h +PASS API/worker->PostgreSQL verify-full, rogue CA as trust expect=fail got=fail server_cert_rejected_by_client +PASS API/worker->PostgreSQL verify-full by IP (SAN mismatch) expect=fail got=fail server_cert_rejected_by_client +PASS API/worker->PostgreSQL verify-full, internal CA, wrong pwd expect=ok got=ok tls_ok_then_password_rejected +PASS API/worker->S3 plaintext HTTP expect=fail got=fail http_400_plain_http_sent_to_https_port +PASS API/worker->S3 HTTPS, rogue CA as trust expect=fail got=fail server_cert_rejected_by_client:unable to get local issuer certificate +PASS API/worker->S3 HTTPS verified, anonymous list expect=ok got=ok 403 +PASS API/worker->broker plaintext AMQP 5672 expect=fail got=fail connection_refused +PASS API/worker->broker AMQPS, rogue CA as trust expect=fail got=fail server_cert_rejected_by_client:self-signed certificate in certificate chain +PASS API/worker->broker AMQPS, client cert of rogue CA expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS API/worker->broker AMQPS verified (без cert), Connection.Start expect=ok got=ok amqp_connection_start +PASS API/worker->broker AMQPS, user worker, wrong password expect=fail got=fail auth_refused +PASS Prometheus->broker metrics HTTPS 15691 without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS Prometheus->broker metrics plaintext 15692 from network expect=fail got=fail connection_refused +PASS API/worker->Collector plaintext expect=fail got=fail http_400_plain_http_sent_to_https_port +PASS API/worker->Collector TLS without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS API/worker->Collector TLS, client cert of rogue CA expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS API/worker->Collector TLS, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'. +PASS Collector/Grafana->Prometheus plaintext expect=fail got=fail http_400_plain_http_sent_to_https_port +PASS Collector/Grafana->Prometheus TLS without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS Collector/Grafana->Prometheus TLS, client cert of rogue CA expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS Collector/Grafana->Prometheus TLS, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'. +PASS Collector/Grafana->Loki plaintext expect=fail got=fail http_400_plain_http_sent_to_https_port +PASS Collector/Grafana->Loki TLS without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS Collector/Grafana->Loki TLS, client cert of rogue CA expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS Collector/Grafana->Loki TLS, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'. +PASS Grafana->Tempo plaintext expect=fail got=fail http_400_plain_http_sent_to_https_port +PASS Grafana->Tempo TLS without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS Grafana->Tempo TLS, client cert of rogue CA expect=fail got=fail tls_alert:SSLEOFError +PASS Grafana->Tempo TLS, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'. +PASS Collector->Tempo OTLP gRPC plaintext expect=fail got=fail closed_without_plaintext_response +PASS Collector->Tempo OTLP gRPC TLS without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS Collector->Tempo OTLP gRPC TLS, client cert of rogue CA expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS Collector->Tempo OTLP gRPC TLS, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'. +PASS Prometheus->LB metrics plaintext expect=fail got=fail closed_without_plaintext_response +PASS Prometheus->LB metrics TLS without client cert expect=fail got=fail tls_alert:TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS Prometheus->LB metrics TLS, client cert of rogue CA expect=fail got=fail tls_alert:TLSV1_ALERT_UNKNOWN_CA +PASS Prometheus->LB metrics TLS, wrong expected hostname expect=fail got=fail server_cert_rejected_by_client:Hostname mismatch, certificate is not valid for 'api'. +PASS internal-only port otel-collector:4317 from network expect=fail got=fail connection_refused +PASS internal-only port otel-collector:8888 from network expect=fail got=fail connection_refused +PASS internal-only port otel-collector:13133 from network expect=fail got=fail connection_refused +PASS internal-only port loki:9095 from network expect=fail got=fail connection_refused +PASS internal-only port tempo:9096 from network expect=fail got=fail connection_refused +INFO API/worker->Collector intruder cert: ('ok', 405) +PASS services->OpenBao plaintext HTTP expect=fail got=fail http_400_plain_http_sent_to_https_port +PASS services->OpenBao TLS, rogue CA as trust expect=fail got=fail server_cert_rejected_by_client:unable to get local issuer certificate +PASS services->OpenBao TLS verified, /v1/sys/health expect=ok got=ok 200 +PASS services->OpenBao cert login as role api with intruder cert expect=fail got=fail http_400 +PASS services->OpenBao cert login as role api with lb cert expect=fail got=fail http_400 +PASS services->OpenBao cert login without client cert expect=fail got=fail http_400 +PASS: 67 checks, 0 failed diff --git a/evidence/load-after-nodelay.json b/evidence/load-after-nodelay.json new file mode 100644 index 0000000..68294e8 --- /dev/null +++ b/evidence/load-after-nodelay.json @@ -0,0 +1,17 @@ +{ + "lab": 7, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.007, + "concurrency": 4, + "successful_requests": 9588, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 319.526, + "successful_latency_ms": { + "p50": 8.292, + "p95": 37.698, + "p99": 77.457 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-before-nodelay.json b/evidence/load-before-nodelay.json new file mode 100644 index 0000000..2c466a8 --- /dev/null +++ b/evidence/load-before-nodelay.json @@ -0,0 +1,17 @@ +{ + "lab": 7, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.029, + "concurrency": 4, + "successful_requests": 2429, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 80.888, + "successful_latency_ms": { + "p50": 48.835, + "p95": 52.869, + "p99": 75.113 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..c3d39fc --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 7, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.008, + "concurrency": 4, + "successful_requests": 9084, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 302.72, + "successful_latency_ms": { + "p50": 7.827, + "p95": 43.767, + "p99": 82.808 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..c3d39fc --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,17 @@ +{ + "lab": 7, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.008, + "concurrency": 4, + "successful_requests": 9084, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 302.72, + "successful_latency_ms": { + "p50": 7.827, + "p95": 43.767, + "p99": 82.808 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/make-check.txt b/evidence/make-check.txt new file mode 100644 index 0000000..9a55330 --- /dev/null +++ b/evidence/make-check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/make-test-security.txt b/evidence/make-test-security.txt new file mode 100644 index 0000000..4e62b8a --- /dev/null +++ b/evidence/make-test-security.txt @@ -0,0 +1,2 @@ +python3 tests/security.py --base-url "https://localhost:8443" --ca-file "/Users/uralazarev/Documents/mtusi2026lab7/secrets/pki/public-ca.crt" +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. diff --git a/evidence/make-test.txt b/evidence/make-test.txt new file mode 100644 index 0000000..e249452 --- /dev/null +++ b/evidence/make-test.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "/Users/uralazarev/Documents/mtusi2026lab7/secrets/pki/public-ca.crt" +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/obs-correlation.json b/evidence/obs-correlation.json new file mode 100644 index 0000000..34e4373 --- /dev/null +++ b/evidence/obs-correlation.json @@ -0,0 +1,238 @@ +{ + "scenario": "lab6 signal correlation: request -> trace -> log, upload -> relay -> worker", + "date_utc": "2026-11-18T20:30:27+00:00", + "steps": [ + { + "t": 0.2, + "utc": "20:30:18.797", + "step": "message sent", + "request_id": "e755c640d7c64606817798996b38e737", + "trace_id": "a5395a7a0d14fddbe988dd3fbc8052c3", + "instance": "r-f7dd6d1932" + }, + { + "t": 2.25, + "utc": "20:30:20.844", + "step": "message trace in Tempo", + "spans": 4, + "server_ms": 5.068, + "db_spans": [ + "db get_session_user", + "db get_chat_for", + "db add_message" + ] + }, + { + "t": 2.28, + "utc": "20:30:20.875", + "step": "structured API log in Loki", + "event": "http.request", + "trace_id": "a5395a7a0d14fddbe988dd3fbc8052c3", + "span_id": "96d2ff4278525fc6", + "severity": "INFO", + "service_name": "messenger-api", + "line": "POST /api/v1/chats/{chat_id}/messages -> 201 4.7ms request_id=e755c640d7c64606817798996b38e737" + }, + { + "t": 2.63, + "utc": "20:30:21.223", + "step": "upload accepted and ready", + "attachment_id": "530ec172-4a69-4d17-b809-7c4f1323c9af", + "trace_id": "f257a257a05eb9b2129200a8a2e3763f", + "status": "ready" + }, + { + "t": 5.78, + "utc": "20:30:24.380", + "step": "one trace upload -> relay -> worker", + "spans": 26, + "services": [ + "messenger-api", + "messenger-relay", + "messenger-worker" + ], + "job_id": "2febd212-9808-4ff5-bf84-f3e5b6f8a705", + "outbox_traceparent": "00-f257a257a05eb9b2129200a8a2e3763f-974c4ff4bf81a0a4-03", + "process_parent_is_send": true, + "send_parent_is_create": true, + "worker_s3_puts": 4, + "api_to_worker_gap_ms": 6.8, + "process_ms": 146.945, + "outcome": "ready" + }, + { + "t": 5.8, + "utc": "20:30:24.393", + "step": "worker structured logs in Loki by trace_id", + "events": [ + "job.claimed", + "job.completed" + ], + "job_id": "2febd212-9808-4ff5-bf84-f3e5b6f8a705", + "span_id": "daa51e4f3cc7737a", + "line": "job 2febd212-9808-4ff5-bf84-f3e5b6f8a705 attempt 1 -> ready in 131ms" + }, + { + "t": 9.21, + "utc": "20:30:27.803", + "step": "no secrets/bodies in telemetry, no ID labels in metrics", + "leaks": { + "loki:message_text": 0, + "loki:password": 0, + "loki:token": 0, + "loki:filename": 0, + "tempo:message_text": 0, + "tempo:password": 0, + "tempo:token": 0, + "tempo:filename": 0 + }, + "metric_labels": [ + "__name__", + "db_client_connection_pool_name", + "db_client_connection_state", + "db_operation_name", + "db_system_name", + "deployment_environment_name", + "error_type", + "http_request_method", + "http_response_status_class", + "http_response_status_code", + "http_route", + "instance", + "job", + "le", + "messenger_job_outcome", + "messenger_job_status", + "messenger_job_type", + "messenger_s3_operation", + "otel_component_name", + "otel_component_type", + "otel_span_parent_origin", + "otel_span_sampling_result", + "outcome", + "reason", + "server_address", + "server_port", + "service_version", + "telemetry_sdk_language", + "telemetry_sdk_name", + "telemetry_sdk_version" + ] + } + ], + "message_trace_tree": [ + "POST /api/v1/chats/{chat_id}/messages messenger-api + 0.0ms 5.1ms SERVER", + " db get_session_user messenger-api + 0.1ms 0.8ms CLIENT", + " db get_chat_for messenger-api + 1.0ms 0.8ms CLIENT", + " db add_message messenger-api + 1.9ms 2.3ms CLIENT" + ], + "message": { + "request_id": "e755c640d7c64606817798996b38e737", + "trace_id": "a5395a7a0d14fddbe988dd3fbc8052c3", + "loki_labels": [ + "code_file_path", + "code_function_name", + "code_line_number", + "deployment_environment_name", + "detected_level", + "duration_ms", + "event", + "flags", + "http_request_method", + "http_response_status_code", + "http_route", + "observed_timestamp", + "request_id", + "scope_name", + "service_instance_id", + "service_name", + "service_version", + "severity_number", + "severity_text", + "span_id", + "telemetry_sdk_language", + "telemetry_sdk_name", + "telemetry_sdk_version", + "trace_id" + ] + }, + "upload_trace_tree": [ + "POST /api/v1/chats/{chat_id}/attachments messenger-api + 0.0ms 15.6ms SERVER", + " db get_session_user messenger-api + 0.1ms 1.0ms CLIENT", + " db get_chat_for messenger-api + 1.2ms 0.8ms CLIENT", + " http.receive_body messenger-api + 2.2ms 0.2ms INTERNAL", + " image.validate messenger-api + 2.5ms 0.9ms INTERNAL", + " db create_pending_attachment messenger-api + 3.5ms 2.6ms CLIENT", + " s3 PutObject messenger-api + 6.2ms 3.3ms CLIENT", + " db enqueue_attachment messenger-api + 9.6ms 5.3ms CLIENT", + " db jobs.enqueue messenger-api + 10.0ms 3.6ms CLIENT", + " attachment.process.v1 create messenger-api + 12.3ms 1.2ms PRODUCER", + " attachments send messenger-relay + 16.4ms 3.9ms PRODUCER", + " attachments.process.v1 process messenger-worker + 22.3ms 146.9ms CONSUMER", + " db jobs.claim messenger-worker + 31.3ms 3.8ms CLIENT", + " db jobs.heartbeat messenger-worker + 36.5ms 0.3ms CLIENT", + " worker.fetch_staging messenger-worker + 37.3ms 2.4ms INTERNAL", + " s3 GetObject messenger-worker + 37.4ms 2.0ms CLIENT", + " db jobs.heartbeat messenger-worker + 39.8ms 0.2ms CLIENT", + " worker.transform messenger-worker + 40.3ms 117.8ms INTERNAL", + " db jobs.heartbeat messenger-worker + 158.4ms 0.3ms CLIENT", + " s3 PutObject messenger-worker + 159.2ms 2.4ms CLIENT", + " s3 PutObject messenger-worker + 161.7ms 0.8ms CLIENT", + " s3 PutObject messenger-worker + 162.6ms 0.8ms CLIENT", + " s3 PutObject messenger-worker + 163.5ms 0.8ms CLIENT", + " db jobs.heartbeat messenger-worker + 164.7ms 0.3ms CLIENT", + " db jobs.complete messenger-worker + 165.3ms 1.6ms CLIENT", + " s3 DeleteObject messenger-worker + 167.8ms 0.9ms CLIENT" + ], + "upload": { + "attachment_id": "530ec172-4a69-4d17-b809-7c4f1323c9af", + "job_id": "2febd212-9808-4ff5-bf84-f3e5b6f8a705", + "trace_id": "f257a257a05eb9b2129200a8a2e3763f", + "worker_log_events": [ + "job.claimed", + "job.completed" + ] + }, + "leaks": { + "loki:message_text": 0, + "loki:password": 0, + "loki:token": 0, + "loki:filename": 0, + "tempo:message_text": 0, + "tempo:password": 0, + "tempo:token": 0, + "tempo:filename": 0 + }, + "metric_labels": [ + "__name__", + "db_client_connection_pool_name", + "db_client_connection_state", + "db_operation_name", + "db_system_name", + "deployment_environment_name", + "error_type", + "http_request_method", + "http_response_status_class", + "http_response_status_code", + "http_route", + "instance", + "job", + "le", + "messenger_job_outcome", + "messenger_job_status", + "messenger_job_type", + "messenger_s3_operation", + "otel_component_name", + "otel_component_type", + "otel_span_parent_origin", + "otel_span_sampling_result", + "outcome", + "reason", + "server_address", + "server_port", + "service_version", + "telemetry_sdk_language", + "telemetry_sdk_name", + "telemetry_sdk_version" + ] +} diff --git a/evidence/obs-correlation.trees.txt b/evidence/obs-correlation.trees.txt new file mode 100644 index 0000000..4cee035 --- /dev/null +++ b/evidence/obs-correlation.trees.txt @@ -0,0 +1,33 @@ +# message trace a5395a7a0d14fddbe988dd3fbc8052c3 +POST /api/v1/chats/{chat_id}/messages messenger-api + 0.0ms 5.1ms SERVER + db get_session_user messenger-api + 0.1ms 0.8ms CLIENT + db get_chat_for messenger-api + 1.0ms 0.8ms CLIENT + db add_message messenger-api + 1.9ms 2.3ms CLIENT + +# upload trace f257a257a05eb9b2129200a8a2e3763f +POST /api/v1/chats/{chat_id}/attachments messenger-api + 0.0ms 15.6ms SERVER + db get_session_user messenger-api + 0.1ms 1.0ms CLIENT + db get_chat_for messenger-api + 1.2ms 0.8ms CLIENT + http.receive_body messenger-api + 2.2ms 0.2ms INTERNAL + image.validate messenger-api + 2.5ms 0.9ms INTERNAL + db create_pending_attachment messenger-api + 3.5ms 2.6ms CLIENT + s3 PutObject messenger-api + 6.2ms 3.3ms CLIENT + db enqueue_attachment messenger-api + 9.6ms 5.3ms CLIENT + db jobs.enqueue messenger-api + 10.0ms 3.6ms CLIENT + attachment.process.v1 create messenger-api + 12.3ms 1.2ms PRODUCER + attachments send messenger-relay + 16.4ms 3.9ms PRODUCER + attachments.process.v1 process messenger-worker + 22.3ms 146.9ms CONSUMER + db jobs.claim messenger-worker + 31.3ms 3.8ms CLIENT + db jobs.heartbeat messenger-worker + 36.5ms 0.3ms CLIENT + worker.fetch_staging messenger-worker + 37.3ms 2.4ms INTERNAL + s3 GetObject messenger-worker + 37.4ms 2.0ms CLIENT + db jobs.heartbeat messenger-worker + 39.8ms 0.2ms CLIENT + worker.transform messenger-worker + 40.3ms 117.8ms INTERNAL + db jobs.heartbeat messenger-worker + 158.4ms 0.3ms CLIENT + s3 PutObject messenger-worker + 159.2ms 2.4ms CLIENT + s3 PutObject messenger-worker + 161.7ms 0.8ms CLIENT + s3 PutObject messenger-worker + 162.6ms 0.8ms CLIENT + s3 PutObject messenger-worker + 163.5ms 0.8ms CLIENT + db jobs.heartbeat messenger-worker + 164.7ms 0.3ms CLIENT + db jobs.complete messenger-worker + 165.3ms 1.6ms CLIENT + s3 DeleteObject messenger-worker + 167.8ms 0.9ms CLIENT diff --git a/evidence/obs-correlation.txt b/evidence/obs-correlation.txt new file mode 100644 index 0000000..9ba859d --- /dev/null +++ b/evidence/obs-correlation.txt @@ -0,0 +1,8 @@ +[ 0.20s] message sent {"request_id": "e755c640d7c64606817798996b38e737", "trace_id": "a5395a7a0d14fddbe988dd3fbc8052c3", "instance": "r-f7dd6d1932"} +[ 2.25s] message trace in Tempo {"spans": 4, "server_ms": 5.068, "db_spans": ["db get_session_user", "db get_chat_for", "db add_message"]} +[ 2.28s] structured API log in Loki {"event": "http.request", "trace_id": "a5395a7a0d14fddbe988dd3fbc8052c3", "span_id": "96d2ff4278525fc6", "severity": "INFO", "service_name": "messenger-api", "line": "POST /api/v1/chats/{chat_id}/messages -> 201 4.7ms request_id=e755c640d7c64606817798996b38e737"} +[ 2.63s] upload accepted and ready {"attachment_id": "530ec172-4a69-4d17-b809-7c4f1323c9af", "trace_id": "f257a257a05eb9b2129200a8a2e3763f", "status": "ready"} +[ 5.78s] one trace upload -> relay -> worker {"spans": 26, "services": ["messenger-api", "messenger-relay", "messenger-worker"], "job_id": "2febd212-9808-4ff5-bf84-f3e5b6f8a705", "outbox_traceparent": "00-f257a257a05eb9b2129200a8a2e3763f-974c4ff4bf81a0a4-03", "process_parent_is_send": true, "send_parent_is_create": true, "worker_s3_puts": 4, "api_to_worker_gap_ms": 6.8, "process_ms": 146.945, "outcome": "ready"} +[ 5.80s] worker structured logs in Loki by trace_id {"events": ["job.claimed", "job.completed"], "job_id": "2febd212-9808-4ff5-bf84-f3e5b6f8a705", "span_id": "daa51e4f3cc7737a", "line": "job 2febd212-9808-4ff5-bf84-f3e5b6f8a705 attempt 1 -> ready in 131ms"} +[ 9.21s] no secrets/bodies in telemetry, no ID labels in metrics {"leaks": {"loki:message_text": 0, "loki:password": 0, "loki:token": 0, "loki:filename": 0, "tempo:message_text": 0, "tempo:password": 0, "tempo:token": 0, "tempo:filename": 0}, "metric_labels": ["__name__", "db_client_connection_pool_name", "db_client_connection_state", "db_operation_name", "db_system_name", "deployment_environment_name", "error_type", "http_request_method", "http_response_status_class", "http_response_status_code", "http_route", "instance", "job", "le", "messenger_job_outcome", "messenger_job_status", "messenger_job_type", "messenger_s3_operation", "otel_component_name", "otel_component_type", "otel_span_parent_origin", "otel_span_sampling_result", "outcome", "reason", "server_address", "server_port", "service_version", "telemetry_sdk_language", "telemetry_sdk_name", "telemetry_sdk_version"]} +PASS: correlation request->trace->log and upload->relay->worker diff --git a/evidence/persistence-ps-after.txt b/evidence/persistence-ps-after.txt new file mode 100644 index 0000000..043101f --- /dev/null +++ b/evidence/persistence-ps-after.txt @@ -0,0 +1,16 @@ +mtusi-lab7-api-1 Up 7 seconds (healthy) +mtusi-lab7-api-2 Up 7 seconds (healthy) +mtusi-lab7-broker-1 Up 46 seconds (healthy) +mtusi-lab7-db-1 Up 47 seconds (healthy) +mtusi-lab7-grafana-1 Up 10 seconds (healthy) +mtusi-lab7-janitor-1 Up Less than a second (health: starting) +mtusi-lab7-lb-1 Up Less than a second (health: starting) +mtusi-lab7-loki-1 Up 47 seconds +mtusi-lab7-openbao-1 Up 46 seconds (healthy) +mtusi-lab7-otel-collector-1 Up 10 seconds +mtusi-lab7-prometheus-1 Up 47 seconds (healthy) +mtusi-lab7-relay-1 Up 9 seconds (healthy) +mtusi-lab7-s3-1 Up 30 seconds (healthy) +mtusi-lab7-tempo-1 Up 47 seconds +mtusi-lab7-worker-1 Up 9 seconds (healthy) +mtusi-lab7-worker-2 Up 9 seconds (healthy) diff --git a/evidence/persistence-ps-before.txt b/evidence/persistence-ps-before.txt new file mode 100644 index 0000000..e0971c7 --- /dev/null +++ b/evidence/persistence-ps-before.txt @@ -0,0 +1,16 @@ +mtusi-lab7-api-1 Up 11 seconds (healthy) +mtusi-lab7-api-2 Up 10 seconds (healthy) +mtusi-lab7-broker-1 Up 12 minutes (healthy) +mtusi-lab7-db-1 Up 8 minutes (healthy) +mtusi-lab7-grafana-1 Up 12 minutes (healthy) +mtusi-lab7-janitor-1 Up 11 seconds (healthy) +mtusi-lab7-lb-1 Up 10 seconds (healthy) +mtusi-lab7-loki-1 Up 12 minutes +mtusi-lab7-openbao-1 Up 2 minutes (healthy) +mtusi-lab7-otel-collector-1 Up 12 minutes +mtusi-lab7-prometheus-1 Up 12 minutes (healthy) +mtusi-lab7-relay-1 Up 12 seconds (healthy) +mtusi-lab7-s3-1 Up 11 minutes (healthy) +mtusi-lab7-tempo-1 Up 12 minutes +mtusi-lab7-worker-1 Up 11 seconds (healthy) +mtusi-lab7-worker-2 Up 10 seconds (healthy) diff --git a/evidence/prometheus-up.txt b/evidence/prometheus-up.txt new file mode 100644 index 0000000..283db2c --- /dev/null +++ b/evidence/prometheus-up.txt @@ -0,0 +1,7 @@ +haproxy lb:8404 1 +loki loki:3100 1 +otelcol-contrib 00cecf88-3117-40c3-aeeb-1c212dc8d9e3 1 +prometheus localhost:9090 1 +rabbitmq broker:15691 1 +rabbitmq-node broker:15691 1 +tempo tempo:3200 1 diff --git a/evidence/run-all.txt b/evidence/run-all.txt new file mode 100644 index 0000000..3973f94 --- /dev/null +++ b/evidence/run-all.txt @@ -0,0 +1,61 @@ +== make-check (2026-11-18T20:26:42Z) + PASS +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +== make-test (2026-11-18T20:26:42Z) + PASS +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +== make-test-security (2026-11-18T20:26:43Z) + PASS +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. +== lab7-tls-matrix (2026-11-18T20:26:43Z) + PASS +PASS: 67 checks, 0 failed +== lab7-api-security (2026-11-18T20:26:44Z) + PASS +PASS: 33 checks, 0 failed +== lab7-openbao (2026-11-18T20:27:21Z) + PASS +PASS: 83 checks, 0 failed +== lab7-hardening (2026-11-18T20:28:13Z) + PASS +PASS: 180 checks, 0 failed +== test-persistence (2026-11-18T20:28:35Z) + PASS +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +== test-failover (2026-11-18T20:29:23Z) + PASS +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + Container mtusi-lab7-api-1 Created + Container mtusi-lab7-api-2 Running + Container mtusi-lab7-worker-1 Running + Container mtusi-lab7-worker-2 Running + Container mtusi-lab7-api-1 Starting + Container mtusi-lab7-api-1 Started + Container mtusi-lab7-worker-2 Waiting + Container mtusi-lab7-worker-1 Waiting + Container mtusi-lab7-api-1 Waiting + Container mtusi-lab7-api-2 Waiting + Container mtusi-lab7-worker-2 Healthy + Container mtusi-lab7-worker-1 Healthy + Container mtusi-lab7-api-1 Healthy + Container mtusi-lab7-api-2 Healthy +== load (2026-11-18T20:29:45Z) + PASS +} +== scan-secrets (2026-11-18T20:30:15Z) + PASS +checked 125 secret values from 50 files; files with leaks: none +== check-rules (2026-11-18T20:30:15Z) + PASS +dashboard OK +== obs-correlation (2026-11-18T20:30:18Z) + PASS +PASS: correlation request->trace->log and upload->relay->worker +== dashboard-check (2026-11-18T20:30:27Z) + PASS +{"dashboard": {"uid": "messenger-overview", "title": "Messenger — наблюдаемость (lab6)", "provisioned": true, "folder": "Messenger", "panels": 56}, "datasource_health": {"prometheus": "OK", "loki": "OK", "tempo": "OK"}, "query_errors": [], "empty_required_panels": [], "pass": true} +== worker-outage (2026-11-18T20:31:09Z) + PASS +PASS: worker outage, api+broker restart, ready, duplicate delivery no-op +== done: ALL PASS +EXIT=0 diff --git a/evidence/scan-secrets.txt b/evidence/scan-secrets.txt new file mode 100644 index 0000000..d993d6c --- /dev/null +++ b/evidence/scan-secrets.txt @@ -0,0 +1 @@ +checked 125 secret values from 50 files; files with leaks: none diff --git a/evidence/test-failover.txt b/evidence/test-failover.txt new file mode 100644 index 0000000..a3f8ab8 --- /dev/null +++ b/evidence/test-failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "/Users/uralazarev/Documents/mtusi2026lab7/secrets/pki/public-ca.crt" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.001} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/test-persistence.txt b/evidence/test-persistence.txt new file mode 100644 index 0000000..6bdfeb2 --- /dev/null +++ b/evidence/test-persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:8443" --ca-file "/Users/uralazarev/Documents/mtusi2026lab7/secrets/pki/public-ca.crt" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/worker-outage.txt b/evidence/worker-outage.txt new file mode 100644 index 0000000..69b6a7d --- /dev/null +++ b/evidence/worker-outage.txt @@ -0,0 +1,11 @@ +[ 1.70s] worker stopped {"running": []} +[ 2.27s] upload while worker is down {"http": 202, "attachment_id": "9628fe62-37bf-4db1-b0a6-304909aba2fd", "status": "queued"} +[ 2.30s] metadata and content while queued {"status": "queued", "content": 409, "thumbnail": 409, "error_code": "conflict"} +[ 2.31s] messages while worker is down {"with_queued_attachment": true, "plain_message_id": "4c5ed20e-1101-48b2-b823-e2f7bac51602"} +[ 7.54s] job is durable in broker {"queue": {"messages": 1, "messages_ready": 1, "messages_unacknowledged": 0, "consumers": 0}, "outbox_published": true, "job_id": "e81a051a-a685-43c4-95f9-0ae1fb1b7027", "job_status": "queued"} +[ 17.67s] api restarted {"status": "queued"} +[ 26.89s] broker restarted with volume {"queue": {"messages": 1, "messages_ready": 1, "messages_unacknowledged": 0, "consumers": 0}, "status": "queued"} +[ 32.84s] worker started +[ 32.90s] ready after worker start {"variants": [["thumbnail", 256, 160], ["crop", 128, 128], ["optimized", 320, 200]], "original_sha256_ok": true, "thumbnail_bytes": 4679} +[ 35.21s] redelivery of the same job is a no-op {"events": ["enqueued", "claimed", "ready", "staging_deleted", "duplicate_ignored"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}} +PASS: worker outage, api+broker restart, ready, duplicate delivery no-op diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..445c5cb --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,3299 @@ +{ + "uid": "messenger-overview", + "title": "Messenger — наблюдаемость (lab6)", + "description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.", + "tags": [ + "messenger", + "lab6", + "otel" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "refresh": "10s", + "schemaVersion": 39, + "version": 1, + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "instance", + "label": "API-реплика", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)", + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "row", + "id": 1, + "title": "Обзор и SLO", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "stat", + "id": 2, + "title": "RPS бизнес-запросов", + "description": "sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_requests:rate2m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 3, + "title": "5xx, доля (2m)", + "description": "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 3, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_error_ratio:rate2m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 4, + "title": "p95 API (5m)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 6, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_latency_seconds:p95_5m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 5, + "title": "SLO доступности, 7d", + "description": "SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 9, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:ratio_7d", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.995 + } + ] + }, + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 6, + "title": "Остаток error budget, 7d", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:error_budget_remaining_7d", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "green", + "value": 0.5 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 7, + "title": "Time to ready p95", + "description": "От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 15, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:time_to_ready_seconds:p95_5m", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 8, + "title": "Ready lag очереди", + "description": "Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 18, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_jobs_ready_lag_seconds) or vector(0)", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "stat", + "id": 9, + "title": "Alerts firing", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 21, + "y": 1, + "w": 3, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)", + "legendFormat": "__auto", + "refId": "A", + "range": false, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + } + }, + { + "type": "state-timeline", + "id": 10, + "title": "История alerts (pending/firing -> resolved)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (alertname, source, alertstate) (ALERTS)", + "legendFormat": "{{alertname}} {{source}} {{alertstate}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "showValue": "never", + "mergeValues": true, + "rowHeight": 0.8, + "legend": { + "showLegend": false + } + } + }, + { + "type": "row", + "id": 11, + "title": "API: трафик, ошибки, задержка", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 11, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 12, + "title": "RPS по маршруту", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 12, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", instance=~\"$instance\"}[1m]))", + "legendFormat": "{{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 13, + "title": "Ответы по классу статуса", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 12, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", instance=~\"$instance\", http_route!~\"/health/.*|unmatched\"}[1m]))", + "legendFormat": "{{http_response_status_class}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 14, + "title": "Доля 5xx: API и LB", + "description": "Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 12, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_error_ratio:rate2m", + "legendFormat": "API (без /health)", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:lb_error_ratio:rate2m", + "legendFormat": "HAProxy fe_api", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 15, + "title": "Задержка API p50/p95/p99 (из histogram)", + "description": "Точки-exemplars ведут в трассу Tempo конкретного запроса.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", instance=~\"$instance\", http_route!~\"/health/.*|unmatched\"}[1m])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", instance=~\"$instance\", http_route!~\"/health/.*|unmatched\"}[1m])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", instance=~\"$instance\", http_route!~\"/health/.*|unmatched\"}[1m])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false, + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 16, + "title": "p95 по маршруту", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", instance=~\"$instance\", http_route!~\"/health/.*|unmatched\"}[5m])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 17, + "title": "5xx по причине (error.type)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", instance=~\"$instance\", http_response_status_class=\"5xx\"}[1m]))", + "legendFormat": "{{error_type}} {{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 18, + "title": "Запросы в обработке (saturation API)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 28, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (http_server_active_requests{job=\"messenger-api\", instance=~\"$instance\"})", + "legendFormat": "{{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 19, + "title": "HAProxy: UP-реплики и очередь LB", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 28, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "haproxy_backend_active_servers{proxy=\"be_api\"}", + "legendFormat": "UP реплик", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "haproxy_backend_current_queue{proxy=\"be_api\"}", + "legendFormat": "очередь LB", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "id": 20, + "title": "Worker и очередь заданий", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 36, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 21, + "title": "Задания по исходу (в минуту)", + "description": "ready / retry / failed / duplicate / lease_lost / rejected / requeued", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 37, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{messenger_job_outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 22, + "title": "Длительность обработки job p50/p95/p99", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 37, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome=\"ready\"}[5m])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome=\"ready\"}[5m])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome=\"ready\"}[5m])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false, + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 23, + "title": "Time to ready p50/p95/p99", + "description": "Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 37, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome=\"ready\"}[5m])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome=\"ready\"}[5m])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false, + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome=\"ready\"}[5m])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false, + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 24, + "title": "Глубина очереди: PostgreSQL и RabbitMQ", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 45, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~\"queued|processing\"})", + "legendFormat": "jobs {{messenger_job_status}} (БД)", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_outbox_pending)", + "legendFormat": "outbox не опубликовано", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rabbitmq_detailed_queue_messages_ready{queue=\"attachments.process.v1\"})", + "legendFormat": "RabbitMQ ready", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rabbitmq_detailed_queue_messages_unacked{queue=\"attachments.process.v1\"})", + "legendFormat": "RabbitMQ unacked", + "refId": "D", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rabbitmq_detailed_queue_messages{queue=\"attachments.dlq\"})", + "legendFormat": "DLQ", + "refId": "E", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 25, + "title": "Возраст старейшего задания и lag", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 45, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)", + "legendFormat": "oldest {{messenger_job_status}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_jobs_ready_lag_seconds)", + "legendFormat": "ready lag", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_outbox_oldest_age_seconds)", + "legendFormat": "outbox oldest", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 26, + "title": "Ожидание в очереди p95 (до claim)", + "description": "Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 45, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", + "legendFormat": "p95 queue wait", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", + "legendFormat": "p95 outbox -> broker", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 27, + "title": "Saturation worker: занятость потребителей", + "description": "1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 53, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", + "legendFormat": "utilization (busy_time/capacity)", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", + "legendFormat": "busy/capacity (мгновенно)", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 28, + "title": "Relay: публикации outbox", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 53, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_outbox_published_total[1m]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "id": 29, + "title": "PostgreSQL и S3", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 61, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 30, + "title": "Пул соединений API: used / idle / max", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 62, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance, db_client_connection_state) (db_client_connection_count{job=\"messenger-api\", instance=~\"$instance\"})", + "legendFormat": "{{instance}} {{db_client_connection_state}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(db_client_connection_max{job=\"messenger-api\", instance=~\"$instance\"})", + "legendFormat": "max на реплику", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 31, + "title": "Ожидающие соединение и таймауты пула", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 62, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (db_client_connection_pending_requests{job=\"messenger-api\", instance=~\"$instance\"})", + "legendFormat": "pending {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (increase(db_client_connection_timeouts_total{job=\"messenger-api\", instance=~\"$instance\"}[1m]))", + "legendFormat": "timeouts/мин {{instance}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 32, + "title": "Ожидание соединения пула p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 62, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{job=\"messenger-api\", instance=~\"$instance\"}[1m])))", + "legendFormat": "p95 wait", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 33, + "title": "Операции БД p95 по операции (API)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 70, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=\"messenger-api\", instance=~\"$instance\"}[1m])))", + "legendFormat": "{{db_operation_name}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 34, + "title": "Операции БД worker/relay p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 70, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~\"messenger-worker|messenger-relay\"}[5m])))", + "legendFormat": "{{db_operation_name}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 35, + "title": "Ошибки операций БД", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 78, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=\"\"}[1m]))", + "legendFormat": "{{job}} {{db_operation_name}} {{error_type}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 36, + "title": "S3 p95 по операции", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 78, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))", + "legendFormat": "{{messenger_s3_operation}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 37, + "title": "S3 ошибки", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 78, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!=\"ok\"}[1m]))", + "legendFormat": "{{job}} {{messenger_s3_operation}} {{outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "id": 38, + "title": "Ресурсы контейнеров (CPU/RAM)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 86, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 39, + "title": "CPU, ядер (cgroup контейнера)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 87, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 40, + "title": "CPU saturation: доля лимита", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 87, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 41, + "title": "Память контейнеров приложений", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 95, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_usage_bytes", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 42, + "title": "Память: доля лимита", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 95, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 43, + "title": "Память telemetry-стека (RSS)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 103, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{job=~\"prometheus|loki|tempo\"}", + "legendFormat": "{{job}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "{__name__=~\"otelcol_process_memory_rss(_bytes)?\"}", + "legendFormat": "otel-collector RSS (вкл. страницы бинарника)", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "otelcol_process_runtime_heap_alloc_bytes", + "legendFormat": "otel-collector Go heap", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rabbitmq_process_resident_memory_bytes", + "legendFormat": "rabbitmq", + "refId": "D", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 44, + "title": "CPU telemetry-стека", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 103, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(process_cpu_seconds_total{job=~\"prometheus|loki|tempo\"}[1m])", + "legendFormat": "{{job}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate({__name__=~\"otelcol_process_cpu_seconds(_total)?\"}[1m])", + "legendFormat": "otel-collector", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "id": 45, + "title": "Конвейер телеметрии (потери)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 111, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 46, + "title": "Collector: принято / отправлено / потеряно (spans)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 112, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\"}[1m]))", + "legendFormat": "принято", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_exporter_sent_spans(_total)?\"}[1m]))", + "legendFormat": "отправлено в Tempo", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_exporter_send_failed_spans(_total)?\"}[1m]))", + "legendFormat": "ошибка отправки", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_exporter_enqueue_failed_spans(_total)?\"}[1m]))", + "legendFormat": "очередь полна (потеря)", + "refId": "D", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_receiver_refused_spans(_total)?\"}[1m]))", + "legendFormat": "отказ приёма (memory_limiter)", + "refId": "E", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 47, + "title": "Collector: логи", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 112, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\"}[1m]))", + "legendFormat": "принято", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_exporter_sent_log_records(_total)?\"}[1m]))", + "legendFormat": "отправлено в Loki", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_exporter_send_failed_log_records(_total)?\"}[1m]))", + "legendFormat": "ошибка отправки", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=~\"otelcol_exporter_enqueue_failed_log_records(_total)?\"}[1m]))", + "legendFormat": "очередь полна (потеря)", + "refId": "D", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 48, + "title": "Collector: очередь экспортёров", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 112, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "size {{exporter}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (exporter) (otelcol_exporter_queue_capacity)", + "legendFormat": "capacity {{exporter}}", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 49, + "title": "SDK приложений: потери на стороне процесса", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 120, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=\"\"}[1m]))", + "legendFormat": "spans dropped {{job}} {{error_type}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=\"\"}[1m]))", + "legendFormat": "logs dropped {{job}} {{error_type}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=\"\"}[1m]))", + "legendFormat": "span export failed {{job}} {{error_type}}", + "refId": "C", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "id": 50, + "title": "SDK: заполнение очереди BatchSpanProcessor", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 120, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (job, instance) (otel_sdk_processor_span_queue_size)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(otel_sdk_processor_span_queue_capacity)", + "legendFormat": "capacity", + "refId": "B", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "id": 51, + "title": "Логи и трассы", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 128, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "logs", + "id": 52, + "title": "Логи WARN/ERROR приложений (Loki)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 129, + "w": 24, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "queryType": "range", + "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN.*|ERROR.*|CRITICAL\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "prettifyLogMessage": false + } + }, + { + "type": "logs", + "id": 53, + "title": "HTTP 5xx (event=http.request)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 139, + "w": 12, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "queryType": "range", + "expr": "{service_name=\"messenger-api\"} | event=\"http.request\" | http_response_status_code >= 500" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "type": "table", + "id": 54, + "title": "Последние трассы обработки заданий (Tempo)", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 12, + "y": 139, + "w": 12, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "limit": 20, + "tableType": "traces", + "query": "{ resource.service.name = \"messenger-worker\" && name = \"attachments.process.v1 process\" }" + } + ] + }, + { + "type": "table", + "id": 55, + "title": "Медленные запросы API > 1 s (Tempo)", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 0, + "y": 149, + "w": 12, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "limit": 20, + "tableType": "traces", + "query": "{ resource.service.name = \"messenger-api\" && kind = server && duration > 1s }" + } + ] + }, + { + "type": "table", + "id": 56, + "title": "Трассы с ошибкой (Tempo)", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 12, + "y": 149, + "w": 12, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "limit": 20, + "tableType": "traces", + "query": "{ status = error }" + } + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..1f1f451 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,15 @@ +# Dashboards из Git: infra/grafana/dashboards/*.json монтируются в /var/lib/grafana/dashboards. +# allowUiUpdates: false — источник истины JSON в репозитории (генератор: scripts/build_dashboard.py). +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Messenger + folderUid: messenger + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..b31bc22 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,103 @@ +# Grafana datasources (provisioning, применяется при старте; правки в UI не сохраняются). +# Grafana — интерфейс и запросы к хранилищам, сама сигналы не хранит. +# Связи сигналов: метрика (exemplar trace_id) -> Tempo; лог (structured metadata trace_id) -> Tempo; +# span -> логи Loki того же trace_id; span -> метрики Prometheus по service.name. +# В provisioning `$` пишется как `$$` (иначе Grafana подставит переменную окружения). +# Lab7: все datasources — HTTPS с проверкой internal-ca и SAN (serverName) и client cert grafana +# (identity только для чтения; Prometheus/Loki/Tempo требуют client cert). Ключ — $__file{} из +# Docker secret, в provisioning/Git его нет. +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: https://prometheus:9090 + isDefault: true + editable: false + secureJsonData: + tlsCACert: $__file{/run/secrets/internal-ca.crt} + tlsClientCert: $__file{/run/secrets/service.crt} + tlsClientKey: $__file{/run/secrets/service.key} + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + tlsSkipVerify: false + serverName: prometheus + httpMethod: POST + timeInterval: 15s + prometheusType: Prometheus + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + uid: loki + type: loki + access: proxy + url: https://loki:3100 + editable: false + secureJsonData: + tlsCACert: $__file{/run/secrets/internal-ca.crt} + tlsClientCert: $__file{/run/secrets/service.crt} + tlsClientKey: $__file{/run/secrets/service.key} + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + tlsSkipVerify: false + serverName: loki + maxLines: 1000 + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: "Trace в Tempo" + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: https://tempo:3200 + editable: false + secureJsonData: + tlsCACert: $__file{/run/secrets/internal-ca.crt} + tlsClientCert: $__file{/run/secrets/service.crt} + tlsClientKey: $__file{/run/secrets/service.key} + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + tlsSkipVerify: false + serverName: tempo + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: true + filterBySpanID: false + customQuery: false + tags: + - key: service.name + value: service_name + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: "-10m" + spanEndTimeShift: "10m" + tags: + - key: service.name + value: job + queries: + - name: "HTTP p95 по маршруту" + query: 'histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{$$__tags}[5m])))' + nodeGraph: + enabled: true + search: + hide: false + lokiSearch: + datasourceUid: loki + traceQuery: + timeShiftEnabled: true + spanStartTimeShift: "-30m" + spanEndTimeShift: "30m" diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..b6cb83a --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,122 @@ +# Lab3/4/6/7: единая точка входа перед API-репликами. +# +# Lab7: снаружи только HTTPS :8443 (сертификат public-ca, SAN localhost/127.0.0.1). LB -> каждая +# API-реплика: mTLS — LB предъявляет identity `lb` (internal-ca), сам проверяет сертификат API +# (internal-ca + SAN api). Readiness-проверки идут тем же mTLS-каналом. Plaintext до API нет. +# +# Lab4: upload вложений — тело идёт потоком (без http-buffer-request); для upload timeout server +# увеличен до 75 с (> UPLOAD_TIMEOUT_SECONDS=60 у API + запись в S3). Content-Length +# больше лимита API отклоняет сам, не читая тело. +# +# Обнаружение реплик: DNS Docker (127.0.0.11) отдаёт по имени сервиса `api` адреса +# ВСЕХ его контейнеров; server-template создаёт до 8 слотов и заполняет их по DNS. +# Поэтому `docker compose up --scale api=N` не требует правки конфигурации. +# +# Readiness: активная проверка GET /health/ready каждую 1 с; 2 неудачи подряд -> DOWN, +# 2 успеха -> UP. Максимальное время обнаружения отказа: +# inter(1s) + timeout check(1s) + fastinter(0.5s) + timeout check(1s) ~= 3.5 с; +# ready=503 или refused без ожидания таймаута -> ~1.5 с. +# Sticky sessions НЕ используются: balance roundrobin без cookie/stick-table для backend +# (stick-table во frontend считает только частоту входов по IP, на выбор реплики не влияет). +# +# Перегрузка: maxconn 10 на реплику (= DB_POOL_SIZE) — лишние запросы ждут в очереди LB +# (timeout queue 5s), а не плодят потоки внутри Python-процесса. Переполнение -> 503 JSON. + +global + log stdout format raw local0 info + maxconn 4096 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305 + ssl-default-server-options ssl-min-ver TLSv1.2 no-tls-tickets + # Без кэша TLS-сессий к backend: каждое новое соединение с API — полный handshake с проверкой. + tune.ssl.cachesize 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold other 2s + hold refused 2s + hold nx 2s + hold timeout 2s + hold obsolete 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + # Внутри одного Docker-host соединение устанавливается за доли мс: 500 мс хватает с запасом, + # а при внезапно исчезнувшей реплике запрос быстро уходит в retry на другую. + timeout connect 500ms + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Повтор только если соединение с backend НЕ установлено (запрос точно не дошёл + # до приложения) — это безопасно даже для неидемпотентных POST. Повтор уходит + # на другую реплику (redispatch). Ответы/таймауты после отправки не повторяются. + retries 2 + retry-on conn-failure + option redispatch 1 + + # Ошибки, которые формирует сам HAProxy (нет живых реплик, обрыв backend, timeout, + # переполнение очереди), отдаются в общем JSON-формате ошибки контракта, а не HTML. + # Ответы приложения (в т.ч. его 503) эти правила не затрагивают. + unique-id-format %[uuid()] + http-error status 400 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"invalid_request","message":"Неверный запрос","request_id":"%ID"}}' + http-error status 408 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"invalid_request","message":"Запрос не получен вовремя","request_id":"%ID"}}' + http-error status 413 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"payload_too_large","message":"Файл больше допустимого размера","request_id":"%ID"}}' + http-error status 429 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "10" lf-string '{"error":{"code":"rate_limited","message":"Слишком много запросов, повторите позже","request_id":"%ID"}}' + http-error status 500 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"internal_error","message":"Внутренняя ошибка сервера","request_id":"%ID"}}' + http-error status 502 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "1" lf-string '{"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"%ID"}}' + http-error status 503 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "1" lf-string '{"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"%ID"}}' + http-error status 504 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "1" lf-string '{"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"%ID"}}' + +frontend fe_api + bind :8443 ssl crt /run/secrets/lb-public.pem alpn http/1.1 + http-response set-header Strict-Transport-Security "max-age=31536000" + # Ответ не раскрывает внутренние адреса/версии LB. + http-response del-header Server + # X-Request-Id формирует само приложение; LB только пробрасывает адрес клиента. + option forwardfor + # Защита входа от перебора паролей и шторма scrypt: не более LOGIN_RATE_LIMIT + # попыток POST /auth/sessions за 10 с с одного IP, сверх — 429 rate_limited. + stick-table type ip size 100k expire 60s store http_req_rate(10s) + acl is_login method POST + acl is_login_path path /api/v1/auth/sessions + # Явно вычисляем unique-id до deny: иначе в ответе 429 request_id остаётся пустым. + http-request set-var(txn.request_id) unique-id + http-request track-sc0 src if is_login is_login_path + http-request deny deny_status 429 if is_login is_login_path { sc_http_req_rate(0) gt "${LOGIN_RATE_LIMIT}" } + # Lab4: лимит размера upload проверяет API по Content-Length ДО чтения тела (ответ 413 + # с X-Instance-Id по контракту). Дублирующий deny на LB убран: его ответ без X-Instance-Id. + default_backend be_api + +backend be_api + balance roundrobin + http-request set-timeout server 75s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ } + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # on-marked-down shutdown-sessions НЕ используется: при штатной остановке реплика ещё + # дообслуживает начатые запросы (drain), и обрыв их сессий дал бы лишние 502. + default-server maxconn 10 check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none ssl verify required ca-file /run/secrets/internal-ca.crt verifyhost api sni str(api) check-sni api crt /run/secrets/lb.pem alpn http/1.1 no-ssl-reuse + server-template api 8 api:8443 + +# Статистика LB и /metrics: только сеть telemetry, HTTPS с обязательным client cert internal-ca +# (Prometheus). Наружу (на хост) не публикуется. +frontend stats + bind :8404 ssl crt /run/secrets/lb.pem ca-file /run/secrets/internal-ca.crt verify required + # Lab6: встроенный Prometheus-экспортёр HAProxy (USE_PROMEX) — снимает Prometheus по сети telemetry. + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 2s + stats show-legends diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..626cc07 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,81 @@ +# Loki (grafana/loki:3.6.19) — хранилище ЛОГОВ, монолитный режим, файловая система (volume lokidata). +# Приём: нативный OTLP /otlp/v1/logs от Collector. Индексные labels — только ресурсные +# service.name, service.instance.id, deployment.environment.name (низкая кардинальность). +# Атрибуты записи (event, request_id, trace_id, span_id, http.route, ...) хранятся как +# structured metadata: по ним можно фильтровать, но они не раздувают индекс. +auth_enabled: false + +# Lab7: HTTP :3100 (OTLP-приём от Collector, запросы Grafana, scrape Prometheus) — только TLS +# с обязательным client cert internal-ca. Внутренний gRPC (querier <-> frontend одного процесса) +# слушает только 127.0.0.1 контейнера. +server: + http_listen_port: 3100 + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9095 + http_tls_config: + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/internal-ca.crt + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + ring: + kvstore: + store: inmemory + +# WAL ингестера — явно на volume: логи, ещё не сброшенные в chunks, переживают рестарт Loki. +ingester: + wal: + enabled: true + dir: /loki/wal + flush_on_shutdown: true + chunk_idle_period: 5m + max_chunk_age: 1h + +schema_config: + configs: + - from: "2024-04-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_query_lookback: 168h + reject_old_samples: true + reject_old_samples_max_age: 24h + volume_enabled: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - deployment.environment.name + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +pattern_ingester: + enabled: false + +ruler: + enable_api: false + +analytics: + reporting_enabled: false diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 0850e23..919cac5 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -1,7 +1,67 @@ -# OpenBao — реализовать конфигурацию и политики +# OpenBao — конфигурация, политики и процедуры оператора (lab7) Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key. +## Что здесь лежит + +| Файл | Назначение | +| --- | --- | +| [openbao.hcl](openbao.hcl) | Сервер **не в dev-mode**: integrated storage Raft (одна нода) на volume `baodata`, TLS-listener `0.0.0.0:8200` (сертификат `openbao` от internal-ca, TLS ≥ 1.2, клиентский сертификат запрашивается для auth/cert), кластерный адрес только `127.0.0.1:8201`, декларативный audit device → stdout (значения HMAC) | +| [policies/messenger-<svc>.hcl](policies/) | По одной policy на сервис: `read` **только** `kv/data/messenger/`. Без list/metadata, без записи, без `sys/*`. Широкого `path "*"` нет | +| ../../scripts/bao-init.sh | Операторский bootstrap (идемпотентный): init → unseal → KV v2 `kv/` → policies → роли `auth/cert/certs/` → запись `kv/messenger/` | +| ../../scripts/bao-unseal.sh | Операторский unseal после рестарта (2 из 3 shares) | + +Почему Raft, а не file: в OpenBao 2.7 file storage удалён (`unknown storage type file`). Почему без mlock: в OpenBao 2.7 параметр убран; swap в Docker Desktop VM отключён, а контейнер работает без `IPC_LOCK` (cap_drop ALL). + +## Аутентификация сервисов — auth/cert + +Identity сервиса — его leaf-сертификат internal-ca (тот же, что для mTLS). Роль на каждый сервис: + +``` +auth/cert/certs/: certificate=internal-ca.crt allowed_dns_sans= token_policies=messenger- + token_type=service token_ttl=15m token_max_ttl=1h (migrate, s3-init: 5m / 10m) +``` + +Сертификат того же CA, но с другим SAN (например `intruder` или `lb`) не подходит ни под одну чужую роль → `400`. В процессе нет root token, `BAO_TOKEN`, SecretID: bootstrap trust = `internal-ca.crt` + собственный ключ (Docker secret). Клиент: [src/vault.py](../../src/vault.py) — login, чтение KV v2, renew-self на ½ TTL, при неудаче — повторный login и перечитывание KV; при недоступном OpenBao работающий процесс сохраняет значения в памяти. + +| Сервис | KV-путь | Поля | +| --- | --- | --- | +| api | `kv/messenger/api` | `POSTGRES_PASSWORD` (роль api_svc), `S3_ACCESS_KEY_ID/SECRET` (ключ api) | +| worker | `kv/messenger/worker` | пароль worker_svc, ключ S3 worker, `BROKER_URL` (пользователь worker, amqps) | +| relay | `kv/messenger/relay` | пароль relay_svc, `BROKER_URL` (пользователь relay) | +| janitor | `kv/messenger/janitor` | пароль janitor_svc, ключ S3 janitor (List+Write) | +| migrate | `kv/messenger/migrate` | пароль суперпользователя `postgres` и пароли всех сервисных ролей (job выставляет их ролям) | +| s3-init | `kv/messenger/s3-init` | ключ S3 admin (только создание bucket) | +| ops | `kv/messenger/ops` | пароль ops_svc, `BROKER_URL` ops, ключ S3 janitor | + +## Процедуры оператора + +**Первый запуск (новый volume `baodata`):** + +```sh +./scripts/init-secrets.sh # пароли + PKI (CA keys -> $OPERATOR_DIR/ca) +./scripts/bao-init.sh # init (3 shares, порог 2) -> $OPERATOR_DIR/openbao/init.json; unseal; policies; роли; KV +docker compose --env-file .env up -d --build --wait +``` + +`$OPERATOR_DIR` по умолчанию `~/.mtusi2026lab7-operator` (700, файлы 600, вне проекта, в контейнеры не монтируется). Скрипт ничего секретного не печатает; root token передаётся только через env процесса `docker compose exec -e BAO_TOKEN`, shares и значения KV — через stdin. + +**После рестарта OpenBao** (рестарт контейнера/хоста, `scripts/restart.sh`): процесс стартует **sealed**, healthcheck `bao status` → unhealthy. + +```sh +./scripts/bao-unseal.sh +``` + +Обычный рестарт API/worker unseal не требует. Пока OpenBao sealed: уже работающие процессы обслуживают запросы; новые ждут (не ready) до 300 с, затем выходят и перезапускаются Docker. + +**Изменить секрет/policy/роль:** поправить `secrets/…` или `policies/*.hcl` → `./scripts/bao-init.sh` (перезапишет) → для паролей ролей БД `docker compose up migrate` → пересоздать затронутый сервис. + +**Отозвать identity сервиса:** с root token (`BAO_TOKEN` из init.json в env): `bao delete auth/cert/certs/`, `bao token revoke -mode=path auth/cert/login`. + +**Потеря volume `baodata`:** все KV пропадут; заново `bao-init.sh` (init.json придётся удалить вручную — скрипт отказывается перезаписывать материал оператора), значения восстанавливаются из `secrets/`. + Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения. +Проверка: `python3 student-tests/lab7_openbao.py` ([evidence/lab7-openbao.json](../../evidence/lab7-openbao.json)) и [src/selfcheck.py](../../src/selfcheck.py), запускаемый внутри каждого сервиса под его identity. + Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md. diff --git a/infra/openbao/openbao.hcl b/infra/openbao/openbao.hcl new file mode 100644 index 0000000..d712ac6 --- /dev/null +++ b/infra/openbao/openbao.hcl @@ -0,0 +1,36 @@ +# OpenBao 2.7.2 — НЕ dev-mode. Integrated storage (Raft, одна нода) на volume baodata: данные +# переживают рестарт; после каждого старта процесс sealed до unseal оператором (scripts/bao-unseal.sh). +# (file storage в OpenBao 2.7 удалён; mlock не используется — swap в Docker VM выключен, см. REPORT.md.) +ui = false +api_addr = "https://openbao:8200" +# Кластерный канал Raft не нужен одной ноде: слушает только loopback контейнера. +cluster_addr = "https://127.0.0.1:8201" +log_level = "info" +default_lease_ttl = "1h" +max_lease_ttl = "4h" + +storage "raft" { + path = "/openbao/file" + node_id = "openbao-1" +} + +# Единственный listener — TLS 1.2+ с сертификатом internal-ca (SAN openbao/localhost). +# tls_client_ca_file: сервер ЗАПРАШИВАЕТ клиентский сертификат (не требует — оператору внутри +# контейнера он не нужен); метод auth/cert проверяет его цепочку и SAN по роли. +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "127.0.0.1:8201" + tls_cert_file = "/run/secrets/service.crt" + tls_key_file = "/run/secrets/service.key" + tls_client_ca_file = "/run/secrets/internal-ca.crt" + tls_min_version = "tls12" +} + +# Аудит (OpenBao 2.7: только декларативно в конфиге). В stdout -> docker logs. Значения +# токенов/секретов в записях аудита — HMAC, не plaintext. +audit "file" "stdout" { + description = "audit to container stdout" + options { + file_path = "stdout" + } +} diff --git a/infra/openbao/policies/messenger-api.hcl b/infra/openbao/policies/messenger-api.hcl new file mode 100644 index 0000000..b2985ad --- /dev/null +++ b/infra/openbao/policies/messenger-api.hcl @@ -0,0 +1,5 @@ +# Policy сервиса api: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/api). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/api" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-janitor.hcl b/infra/openbao/policies/messenger-janitor.hcl new file mode 100644 index 0000000..a4a8015 --- /dev/null +++ b/infra/openbao/policies/messenger-janitor.hcl @@ -0,0 +1,5 @@ +# Policy сервиса janitor: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/janitor). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/janitor" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-migrate.hcl b/infra/openbao/policies/messenger-migrate.hcl new file mode 100644 index 0000000..4d3dd37 --- /dev/null +++ b/infra/openbao/policies/messenger-migrate.hcl @@ -0,0 +1,5 @@ +# Policy сервиса migrate: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/migrate). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/migrate" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-ops.hcl b/infra/openbao/policies/messenger-ops.hcl new file mode 100644 index 0000000..bed3272 --- /dev/null +++ b/infra/openbao/policies/messenger-ops.hcl @@ -0,0 +1,5 @@ +# Policy сервиса ops: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/ops). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/ops" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-relay.hcl b/infra/openbao/policies/messenger-relay.hcl new file mode 100644 index 0000000..c124085 --- /dev/null +++ b/infra/openbao/policies/messenger-relay.hcl @@ -0,0 +1,5 @@ +# Policy сервиса relay: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/relay). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/relay" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-s3-init.hcl b/infra/openbao/policies/messenger-s3-init.hcl new file mode 100644 index 0000000..20abbe5 --- /dev/null +++ b/infra/openbao/policies/messenger-s3-init.hcl @@ -0,0 +1,5 @@ +# Policy сервиса s3-init: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/s3-init). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/s3-init" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-worker.hcl b/infra/openbao/policies/messenger-worker.hcl new file mode 100644 index 0000000..33e12f4 --- /dev/null +++ b/infra/openbao/policies/messenger-worker.hcl @@ -0,0 +1,5 @@ +# Policy сервиса worker: ТОЛЬКО чтение своего секрета KV v2 (mount kv/, путь messenger/worker). +# Нет list/metadata, нет записи, нет sys/*, нет чужих путей. renew-self/lookup-self даёт default. +path "kv/data/messenger/worker" { + capabilities = ["read"] +} diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..f83528a 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -5,3 +5,18 @@ Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. + +## Решение lab7 + +- [collector.yaml](collector.yaml): приём только OTLP/HTTPS `:4318` с **обязательным** client cert internal-ca + (`client_ca_file`); gRPC 4317 выключен; собственные метрики (`127.0.0.1:8888`) и `health_check` + (`127.0.0.1:13133`) — только loopback, метрики Collector попадают в Prometheus через receiver + `prometheus/self` и общий pipeline (job `otelcol-contrib`). +- Экспорт: Tempo OTLP gRPC, Loki `/otlp`, Prometheus `/api/v1/otlp` — TLS с проверкой CA и + `server_name_override`, client cert `otel-collector` (backends требуют его). +- Scrub (`transform/scrub`): Authorization/Cookie/url.query/параметры SQL и всё похожее на + password/secret/token удаляется из атрибутов spans/логов, Bearer/Basic в теле лога маскируются. +- Prometheus — [web.yml](../prometheus/web.yml) (`RequireAndVerifyClientCert`), scrape только HTTPS; + Loki/Tempo — `server.http_tls_config` + внутренний gRPC на 127.0.0.1; Grafana datasources — + HTTPS + client cert `grafana` (`$__file{}` из Docker secret). Проверка: `scripts/check_rules.sh`, + `student-tests/lab7_tls_matrix.py`, `student-tests/dashboard_check.py`. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..7c2d0b6 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,169 @@ +# OpenTelemetry Collector (otel/opentelemetry-collector-contrib:0.163.2) — получатель, обработчик +# и экспортёр телеметрии. Долговременно ничего не хранит: буфер только в памяти и ограничен. +# +# API/relay/worker/janitor --OTLP/HTTP :4318--> receivers.otlp +# -> memory_limiter (первым: отказ приёма при давлении памяти, а не OOM) +# -> resource/lab, transform/scrub (защита в глубину от секретов в атрибутах) +# -> batch +# -> exporters: traces -> Tempo (OTLP gRPC), logs -> Loki (/otlp), metrics -> Prometheus (/api/v1/otlp) +# +# Поведение при недоступном backend: у каждого экспортёра своя ограниченная очередь +# (sending_queue.queue_size батчей, в памяти) и retry с экспоненциальной паузой до 60 с. +# Переполнение очереди или исчерпание retry = потеря данных, она считается в собственных +# метриках Collector (otelcol_exporter_enqueue_failed_*, otelcol_exporter_send_failed_*), +# которые Prometheus снимает с :8888. Приложение от этого не зависит: SDK отправляет асинхронно. +# Lab7 (TLS/mTLS): +# * приём OTLP/HTTPS :4318 — TLS 1.2+, сертификат otel-collector (internal-ca) и ОБЯЗАТЕЛЬНЫЙ +# клиентский сертификат internal-ca (client_ca_file): без него произвольный ingest невозможен; +# OTLP gRPC :4317 выключен (приложения его не используют — меньше поверхность); +# * экспорт в Tempo/Loki/Prometheus — TLS с проверкой CA+SAN и client cert otel-collector (mTLS); +# * собственные метрики Collector (:8888) слушают только 127.0.0.1 контейнера и забираются +# receiver-ом prometheus/self -> общий pipeline -> Prometheus по mTLS (раньше Prometheus +# скрейпил :8888 по plaintext); health_check — тоже 127.0.0.1. +# На хост ничего не публикуется. + +extensions: + health_check: + endpoint: 127.0.0.1:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 8388608 # 8 MiB на запрос экспорта + tls: + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + client_ca_file: /run/secrets/internal-ca.crt + min_version: "1.2" + + # Собственная телеметрия Collector с loopback (job/instance сохраняются как labels в Prometheus). + prometheus/self: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 15s + static_configs: + - targets: ["127.0.0.1:8888"] + +processors: + # Мягкий предел 150 MiB (limit - spike), жёсткий 190 MiB при лимите контейнера 256 MiB + # и GOMEMLIMIT 200MiB. Выше мягкого — receiver отвечает ошибкой (SDK повторит/отбросит). + memory_limiter: + check_interval: 1s + limit_mib: 190 + spike_limit_mib: 40 + + resource/lab: + attributes: + - key: deployment.environment.name + value: lab + action: insert + + # Атрибуты, которых в нашей телеметрии быть не должно. Код их не пишет; процессор гарантирует, + # что случайно добавленный заголовок/секрет не дойдёт до хранилищ. + transform/scrub: + error_mode: ignore + trace_statements: + - delete_key(span.attributes, "http.request.header.authorization") + - delete_key(span.attributes, "http.request.header.cookie") + - delete_key(span.attributes, "url.query") + - delete_key(span.attributes, "db.query.parameters") + - delete_matching_keys(span.attributes, "(?i).*(password|secret|token|authorization).*") + log_statements: + - delete_key(log.attributes, "http.request.header.authorization") + - delete_key(log.attributes, "url.query") + - delete_matching_keys(log.attributes, "(?i).*(password|secret|token|authorization).*") + - replace_pattern(log.body, "(?i)(bearer|basic)\\s+[A-Za-z0-9._~+/=-]{6,}", "$$1 [REDACTED]") + + batch: + send_batch_size: 512 + send_batch_max_size: 1024 + timeout: 1s + +exporters: + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name_override: tempo + min_version: "1.2" + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + otlp_http/loki: + endpoint: https://loki:3100/otlp + tls: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name_override: loki + min_version: "1.2" + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + otlp_http/prometheus: + endpoint: https://prometheus:9090/api/v1/otlp + tls: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name_override: prometheus + min_version: "1.2" + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 50 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + encoding: json + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, resource/lab, transform/scrub, batch] + exporters: [otlp_grpc/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, resource/lab, transform/scrub, batch] + exporters: [otlp_http/loki] + metrics: + receivers: [otlp, prometheus/self] + processors: [memory_limiter, resource/lab, batch] + exporters: [otlp_http/prometheus] diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..53eec0b --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,12 @@ +# Lab7: только TLS. Локальный сокет внутри контейнера — peer (ОС-пользователь postgres = роль +# postgres: initdb-скрипты образа, pg_isready, оператор через docker compose exec). +# TYPE DATABASE USER ADDRESS METHOD +local all postgres peer +local all all reject +# Сеть: только hostssl + SCRAM-SHA-256. Любое подключение без TLS отвергается. +hostssl messenger postgres 0.0.0.0/0 scram-sha-256 +hostssl messenger api_svc,worker_svc,relay_svc,janitor_svc,ops_svc 0.0.0.0/0 scram-sha-256 +hostnossl all all 0.0.0.0/0 reject +hostnossl all all ::/0 reject +host all all 0.0.0.0/0 reject +host all all ::/0 reject diff --git a/infra/postgres/start.sh b/infra/postgres/start.sh new file mode 100755 index 0000000..652529b --- /dev/null +++ b/infra/postgres/start.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Lab7: PostgreSQL требует, чтобы ключ TLS принадлежал postgres (или root) с правами 0600/0640, +# а Docker secret — bind mount с правами хоста. Копируем сертификат/ключ в tmpfs /run/pgtls с +# нужными правами (выполняется root-ом entrypoint-а, затем образ сам переходит на uid postgres). +set -eu +install -d -m 0700 -o postgres -g postgres /run/pgtls +install -m 0600 -o postgres -g postgres /run/secrets/service.key /run/pgtls/server.key +install -m 0644 -o postgres -g postgres /run/secrets/service.crt /run/pgtls/server.crt +exec docker-entrypoint.sh postgres \ + -c ssl=on \ + -c ssl_cert_file=/run/pgtls/server.crt \ + -c ssl_key_file=/run/pgtls/server.key \ + -c ssl_min_protocol_version=TLSv1.2 \ + -c password_encryption=scram-sha-256 \ + -c hba_file=/etc/postgresql/pg_hba.conf \ + -c log_connections=on \ + -c log_hostname=off diff --git a/infra/prometheus/client.yml b/infra/prometheus/client.yml new file mode 100644 index 0000000..f77b8ad --- /dev/null +++ b/infra/prometheus/client.yml @@ -0,0 +1,6 @@ +# Lab7: HTTP-клиент promtool для Docker healthcheck (тот же client cert Prometheus). +tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: prometheus diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..fb624e3 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,97 @@ +# Prometheus (prom/prometheus:v3.13.5) — хранилище МЕТРИК (TSDB на volume promdata, 15 дней / 2 GB). +# Источники: +# * OTLP push от Collector: /api/v1/otlp (флаг --web.enable-otlp-receiver) — метрики API, +# worker, relay, janitor и (lab7) собственные метрики Collector (job=otel-collector). +# job = service.name, instance = service.instance.id (= X-Instance-Id); +# * scrape (lab7 — только HTTPS + client cert prometheus, проверка CA и SAN цели): HAProxy +# (lb:8404/metrics), RabbitMQ (broker:15691), Loki, Tempo и сам Prometheus. +# Alert rules — rules/*.yml (локальное состояние firing/resolved, без Alertmanager: +# доставка наружу по заданию не нужна). +global: + scrape_interval: 15s + scrape_timeout: 5s + evaluation_interval: 15s + +otlp: + # Ресурсные атрибуты, которые станут labels всех OTLP-метрик (низкая кардинальность). + # service.name -> job, service.instance.id -> instance (Prometheus делает сам). + promote_resource_attributes: + - service.version + - deployment.environment.name + +storage: + tsdb: + # OTLP-метрики приходят push-ом от нескольких процессов: допускаем небольшой разброс времени. + out_of_order_time_window: 10m + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: prometheus + scheme: https + tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: haproxy + metrics_path: /metrics + scheme: https + tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: lb + static_configs: + - targets: ["lb:8404"] + + # Глубина очередей брокера (ready/unacked/consumers) по vhost messenger. + - job_name: rabbitmq + metrics_path: /metrics/detailed + params: + vhost: [messenger] + family: [queue_coarse_metrics, queue_consumer_count] + scheme: https + tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: broker + static_configs: + - targets: ["broker:15691"] + + # Агрегированные метрики узла RabbitMQ (память, алармы, соединения). + - job_name: rabbitmq-node + metrics_path: /metrics + scheme: https + tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: broker + static_configs: + - targets: ["broker:15691"] + + - job_name: loki + scheme: https + tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: loki + static_configs: + - targets: ["loki:3100"] + + - job_name: tempo + scheme: https + tls_config: + ca_file: /run/secrets/internal-ca.crt + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + server_name: tempo + static_configs: + - targets: ["tempo:3200"] diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..e39ff1a --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,158 @@ +# Recording и alert rules мессенджера (Prometheus 3.13). Проверка: promtool check rules, +# unit-тесты — infra/prometheus/tests/messenger.test.yml (scripts/check_rules.sh). +# Состояние alerts видно локально: Prometheus /alerts, /api/v1/alerts, ряд ALERTS{alertstate}, +# Grafana -> Alerting (data source-managed) и панель «Alerts» дашборда. Доставки наружу нет. +# +# SLI (бизнес-запросы = все маршруты, кроме /health/* и unmatched): +# доступность = доля ответов не 5xx; SLO 99.5 % за 7 дней (rolling) +# задержка = доля запросов не дольше 0.5 с (без upload +# и входа/регистрации: scrypt by design); SLO 99 % за 7 дней +# time to ready = доля вложений, ready не позже 30 с после +# durable acceptance (COMMIT enqueue); SLO 99 % за 7 дней +# Числитель ошибок — только 5xx (отказ сервера). 4xx (неверный пароль, 404, 409) — корректные +# ответы на ошибку клиента и бюджет не расходуют. Error budget доступности = 0.5 % запросов. +groups: + - name: messenger-sli + interval: 15s + rules: + - record: messenger:http_requests:rate2m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched"}[2m])) + - record: messenger:http_errors:rate2m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched", http_response_status_class="5xx"}[2m])) or vector(0) + - record: messenger:http_error_ratio:rate2m + expr: messenger:http_errors:rate2m / messenger:http_requests:rate2m + - record: messenger:http_requests:rate5m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched"}[5m])) + - record: messenger:http_error_ratio:rate5m + expr: (sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched", http_response_status_class="5xx"}[5m])) or vector(0)) / messenger:http_requests:rate5m + # p50/p95/p99 из buckets ВСЕХ реплик (сначала sum by le, потом quantile) — не среднее p95. + - record: messenger:http_latency_seconds:p50_5m + expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*|unmatched"}[5m]))) + - record: messenger:http_latency_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*|unmatched"}[5m]))) + - record: messenger:http_latency_seconds:p99_5m + expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*|unmatched"}[5m]))) + # Ошибки на входе (HAProxy): если упала БД, реплики не ready и LB сам отвечает 503 — + # до API такие запросы не доходят, и метрика API их не видит. + - record: messenger:lb_requests:rate2m + expr: sum(rate(haproxy_frontend_http_responses_total{proxy="fe_api"}[2m])) + - record: messenger:lb_error_ratio:rate2m + expr: (sum(rate(haproxy_frontend_http_responses_total{proxy="fe_api", code="5xx"}[2m])) or vector(0)) / messenger:lb_requests:rate2m + - record: messenger:db_operation_seconds:p95_2m + expr: histogram_quantile(0.95, sum by (le) (rate(db_client_operation_duration_seconds_bucket{job="messenger-api"}[2m]))) + - record: messenger:time_to_ready_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m]))) + + - name: messenger-slo + interval: 1m + rules: + - record: messenger:slo_availability:ratio_7d + expr: 1 - ((sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched", http_response_status_class="5xx"}[7d])) or vector(0)) / sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched"}[7d]))) + - record: messenger:slo_availability:error_budget_remaining_7d + expr: 1 - (1 - messenger:slo_availability:ratio_7d) / (1 - 0.995) + - record: messenger:slo_latency:ratio_7d + expr: sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api", le="0.5", http_route!~"/health/.*|unmatched|/api/v1/chats/\\{chat_id\\}/attachments|/api/v1/users|/api/v1/auth/sessions"}[7d])) / sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*|unmatched|/api/v1/chats/\\{chat_id\\}/attachments|/api/v1/users|/api/v1/auth/sessions"}[7d])) + - record: messenger:slo_time_to_ready:ratio_7d + expr: sum(increase(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready", le="30"}[7d])) / sum(increase(messenger_attachment_time_to_ready_seconds_count{outcome="ready"}[7d])) + + - name: messenger-alerts + interval: 15s + rules: + - alert: MessengerHighErrorRate + # Окно 2 м, порог 5 % 5xx, минимум 0.1 запроса/с (не будить на единичной ошибке ночью), for 1m. + expr: messenger:http_error_ratio:rate2m > 0.05 and on() messenger:http_requests:rate2m > 0.1 + for: 1m + labels: + severity: page + source: api + annotations: + summary: "API: {{ $value | humanizePercentage }} ответов 5xx за 2 минуты" + description: "Доля 5xx среди бизнес-запросов (без /health) выше 5 % дольше минуты. Error budget SLO 99.5 % расходуется в ~10 раз быстрее допустимого." + runbook_url: "infra/runbooks/README.md#messengerhigherrorrate" + - alert: MessengerHighErrorRate + expr: messenger:lb_error_ratio:rate2m > 0.05 and on() messenger:lb_requests:rate2m > 0.1 + for: 1m + labels: + severity: page + source: lb + annotations: + summary: "LB: {{ $value | humanizePercentage }} ответов 5xx за 2 минуты" + description: "HAProxy отвечает 5xx (вероятно, нет ready-реплик: проверьте /health/ready и БД)." + runbook_url: "infra/runbooks/README.md#messengerhigherrorrate" + + - alert: MessengerQueueLagGrowing + # Самое старое «созревшее» задание ждёт worker дольше 30 с, и ожидание не уменьшается. + expr: max(messenger_jobs_ready_lag_seconds) > 30 and on() deriv(max(messenger_jobs_ready_lag_seconds)[2m:15s]) >= 0 + for: 1m + labels: + severity: page + annotations: + summary: "Очередь обработки вложений растёт: старейшее задание ждёт {{ $value | humanizeDuration }}" + description: "messenger_jobs_ready_lag_seconds > 30 с дольше минуты. Вероятно, worker остановлены/перегружены или брокер недоступен." + runbook_url: "infra/runbooks/README.md#messengerqueuelaggrowing" + - alert: MessengerOutboxStuck + expr: max(messenger_outbox_oldest_age_seconds) > 30 + for: 1m + labels: + severity: page + annotations: + summary: "Outbox не публикуется {{ $value | humanizeDuration }}" + description: "Задания приняты (COMMIT), но relay не может опубликовать их в RabbitMQ." + runbook_url: "infra/runbooks/README.md#messengeroutboxstuck" + - alert: MessengerWorkerSaturated + # last_over_time[30s]: только живые реплики (ряды остановленных контейнеров видны ещё 5 мин). + expr: sum(rate(messenger_worker_busy_time_seconds_total[2m])) / sum(last_over_time(messenger_worker_capacity[30s])) > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "Worker заняты на {{ $value | humanizePercentage }} дольше 5 минут" + runbook_url: "infra/runbooks/README.md#messengerworkersaturated" + + - alert: MessengerDbSlow + # p95 логической операции БД в API выше 250 мс (норма — единицы мс). + expr: messenger:db_operation_seconds:p95_2m > 0.25 + for: 1m + labels: + severity: page + annotations: + summary: "PostgreSQL медленная: p95 операции {{ $value | humanizeDuration }}" + description: "Задержка в spans/гистограмме db.client.operation.duration, а не в коде обработчика. Проверьте блокировки и ресурсы БД." + runbook_url: "infra/runbooks/README.md#messengerdbslow" + - alert: MessengerDbPoolSaturated + expr: max(avg_over_time(db_client_connection_pending_requests{job="messenger-api"}[1m])) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Пул соединений API исчерпан: в среднем {{ $value }} запросов ждут соединение" + runbook_url: "infra/runbooks/README.md#messengerdbpoolsaturated" + - alert: MessengerApiLatencyHigh + expr: messenger:http_latency_seconds:p95_5m > 1 + for: 2m + labels: + severity: warning + annotations: + summary: "p95 бизнес-запросов API {{ $value | humanizeDuration }}" + runbook_url: "infra/runbooks/README.md#messengerapilatencyhigh" + + - alert: MessengerTelemetryExportFailing + # Collector не смог отправить/поставить в очередь данные в backend: телеметрия теряется. + # Collector 0.163 публикует счётчики без суффикса _total — regex по __name__ покрывает оба варианта. + expr: sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[2m]) or vector(0)) + sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[2m]) or vector(0)) + sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[2m]) or vector(0)) + sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[2m]) or vector(0)) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Collector теряет телеметрию ({{ $value | printf \"%.1f\" }} записей/с)" + runbook_url: "infra/runbooks/README.md#messengertelemetryexportfailing" + - alert: MessengerTelemetryBackendDown + # Lab7: метрики Collector приходят push-ом через его же pipeline (receiver prometheus/self), + # поэтому упавший Collector виден как ОТСУТСТВИЕ его up, а не up == 0. + expr: up{job=~"loki|tempo"} == 0 or absent_over_time(up{job="otelcol-contrib"}[2m]) + for: 1m + labels: + severity: warning + annotations: + summary: "Telemetry backend {{ $labels.job }} недоступен" + runbook_url: "infra/runbooks/README.md#messengertelemetrybackenddown" diff --git a/infra/prometheus/tests/messenger.test.yml b/infra/prometheus/tests/messenger.test.yml new file mode 100644 index 0000000..b0a7820 --- /dev/null +++ b/infra/prometheus/tests/messenger.test.yml @@ -0,0 +1,82 @@ +# Unit-тесты alert rules: promtool test rules (запуск — scripts/check_rules.sh, в контейнере +# prom/prometheus той же версии, что и стенд). Синтетические ряды с шагом 15 с. +rule_files: + - ../rules/messenger.yml + +evaluation_interval: 15s + +tests: + # 1) Высокая доля 5xx в API: 4 rps успешных, с t=165 с ещё ~0.67 rps 5xx (≈14 %). + - interval: 15s + input_series: + - series: 'http_server_request_duration_seconds_count{job="messenger-api", instance="a", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}' + values: '0+60x60' + - series: 'http_server_request_duration_seconds_count{job="messenger-api", instance="a", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="5xx"}' + values: '0x10 10+10x20 310x29' + # health не входит в SLI: тысячи 5xx /health/ready не должны поднимать alert. + - series: 'http_server_request_duration_seconds_count{job="messenger-api", instance="a", http_route="/health/ready", http_response_status_class="5xx"}' + values: '0+1000x60' + alert_rule_test: + - eval_time: 2m + alertname: MessengerHighErrorRate + exp_alerts: [] + - eval_time: 5m + alertname: MessengerHighErrorRate + exp_alerts: + - exp_labels: + severity: page + source: api + exp_annotations: + summary: "API: 14.29% ответов 5xx за 2 минуты" + description: "Доля 5xx среди бизнес-запросов (без /health) выше 5 % дольше минуты. Error budget SLO 99.5 % расходуется в ~10 раз быстрее допустимого." + runbook_url: "infra/runbooks/README.md#messengerhigherrorrate" + # 5xx прекратились в t=465 с: через 2-минутное окно alert снят (resolved). + - eval_time: 12m + alertname: MessengerHighErrorRate + exp_alerts: [] + + # 2) Worker остановлен: возраст старейшего созревшего задания растёт с t=135 с, после + # восстановления (t=330 с) — 0. + - interval: 15s + input_series: + - series: 'messenger_jobs_ready_lag_seconds{job="messenger-relay", instance="r"}' + values: '0x8 15+15x12 0x30' + alert_rule_test: + - eval_time: 2m + alertname: MessengerQueueLagGrowing + exp_alerts: [] + - eval_time: 5m + alertname: MessengerQueueLagGrowing + exp_alerts: + - exp_labels: + severity: page + exp_annotations: + summary: "Очередь обработки вложений растёт: старейшее задание ждёт 3m 0s" + description: "messenger_jobs_ready_lag_seconds > 30 с дольше минуты. Вероятно, worker остановлены/перегружены или брокер недоступен." + runbook_url: "infra/runbooks/README.md#messengerqueuelaggrowing" + - eval_time: 9m + alertname: MessengerQueueLagGrowing + exp_alerts: [] + + # 3) Медленная БД: все операции попадают в bucket 1 с (p95 ≈ 0.95 с) с t=150 с. + - interval: 15s + input_series: + - series: 'db_client_operation_duration_seconds_bucket{job="messenger-api", instance="a", le="0.01"}' + values: '0+100x10 1000x30' + - series: 'db_client_operation_duration_seconds_bucket{job="messenger-api", instance="a", le="1"}' + values: '0+100x10 1000+100x30' + - series: 'db_client_operation_duration_seconds_bucket{job="messenger-api", instance="a", le="+Inf"}' + values: '0+100x10 1000+100x30' + alert_rule_test: + - eval_time: 2m + alertname: MessengerDbSlow + exp_alerts: [] + - eval_time: 6m + alertname: MessengerDbSlow + exp_alerts: + - exp_labels: + severity: page + exp_annotations: + summary: "PostgreSQL медленная: p95 операции 950.5ms" + description: "Задержка в spans/гистограмме db.client.operation.duration, а не в коде обработчика. Проверьте блокировки и ресурсы БД." + runbook_url: "infra/runbooks/README.md#messengerdbslow" diff --git a/infra/prometheus/web.yml b/infra/prometheus/web.yml new file mode 100644 index 0000000..9a03ff8 --- /dev/null +++ b/infra/prometheus/web.yml @@ -0,0 +1,8 @@ +# Lab7: HTTP API Prometheus (OTLP-приём от Collector, запросы Grafana, self-scrape) — только TLS +# с обязательным клиентским сертификатом internal-ca. Без сертификата — отказ на handshake. +tls_server_config: + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/internal-ca.crt + min_version: TLS12 diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..708588d --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,30 @@ +# RabbitMQ 4.1 (lab7). Топология и пользователи — из definitions (Docker secret, только хеши паролей). +load_definitions = /run/secrets/rabbitmq_definitions.json +loopback_users.guest = true +# Lab7: plaintext AMQP выключен, только AMQPS 5671 (TLS 1.2/1.3). +listeners.tcp = none +listeners.ssl.default = 5671 +ssl_options.cacertfile = /run/secrets/internal-ca.crt +ssl_options.certfile = /run/secrets/service.crt +ssl_options.keyfile = /run/secrets/service.key +# verify_peer: предъявленный клиентский сертификат проверяется по internal-ca (relay/worker/ops +# предъявляют свои). Авторизация на 3 — пользователь/пароль + ACL vhost (см. definitions). +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = false +ssl_options.versions.1 = tlsv1.3 +ssl_options.versions.2 = tlsv1.2 +# Метрики rabbitmq_prometheus — только HTTPS :15691 с обязательным client cert (Prometheus). +prometheus.tcp.port = 15692 +prometheus.tcp.ip = 127.0.0.1 +prometheus.ssl.port = 15691 +prometheus.ssl.cacertfile = /run/secrets/internal-ca.crt +prometheus.ssl.certfile = /run/secrets/service.crt +prometheus.ssl.keyfile = /run/secrets/service.key +prometheus.ssl.verify = verify_peer +prometheus.ssl.fail_if_no_peer_cert = true +vm_memory_high_watermark.relative = 0.6 +disk_free_limit.absolute = 256MB +consumer_timeout = 1800000 +log.console = true +log.console.level = info +log.file = false diff --git a/infra/runbooks/README.md b/infra/runbooks/README.md new file mode 100644 index 0000000..e10c406 --- /dev/null +++ b/infra/runbooks/README.md @@ -0,0 +1,128 @@ +# Runbooks alerts мессенджера (lab6) + +Alerts вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`), состояние видно локально: +Grafana → дашборд «Messenger — наблюдаемость (lab6)» (панели «Alerts firing» и «История alerts»), +Grafana → Alerting → Alert rules (data source-managed), либо запросом `ALERTS{alertstate="firing"}`. +Доставки наружу нет (Alertmanager не используется — по заданию достаточно firing/resolved). + +Общий порядок диагностики (метрика → трасса → лог): + +1. **Метрика** отвечает «что и насколько плохо»: какая доля ошибок, какой перцентиль, где насыщение. +2. **Трасса** отвечает «где именно тратится время/возникает ошибка»: какой span длинный/с ошибкой — + код обработчика, БД, S3, ожидание в очереди (разрыв между `attachments send` и `process`). +3. **Лог** отвечает «что конкретно произошло»: `event`, `error.type`, `request_id`, `job_id`, + и по `trace_id` ведёт обратно в трассу (в Grafana — ссылка «Trace в Tempo» у записи Loki). + +Команды ниже выполняются из корня репозитория. Доступ к Grafana: , +пользователь `admin`, пароль в `secrets/grafana_admin_password` (не печатать в отчёты). + +--- + +## MessengerHighErrorRate + +**Условие:** доля 5xx среди бизнес-запросов API (без `/health/*` и `unmatched`) за 2 мин > 5 % при +≥ 0.1 запроса/с, `for: 1m` (`source=api`). Вторая копия (`source=lb`) — то же по HAProxy `fe_api`: +если реплики не ready, 503 отдаёт сам LB, и до API запросы не доходят. + +**Смысл для SLO:** при 5 % ошибок бюджет 0.5 % (SLO 99.5 % / 7 дней) расходуется в 10 раз быстрее. + +**Диагностика:** +1. Панель «5xx по причине (error.type)»: `db_unavailable`/`db_pool_timeout` → БД (см. MessengerDbSlow), + `storage_unavailable` → S3, `overloaded` → лимиты слотов (scrypt/upload), `internal_error` → баг. + PromQL: `sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{job="messenger-api",http_response_status_class="5xx"}[1m]))`. +2. `source=lb` и API почти без трафика → `haproxy_backend_active_servers{proxy="be_api"}`; если 0 — + реплики не ready: `docker compose ps api`, `curl -s localhost:8080/health/ready`, затем БД. +3. Трасса ошибки: Grafana → Explore → Tempo → TraceQL + `{ resource.service.name = "messenger-api" && kind = server && status = error }`; длинный + `db <операция>` с ошибкой — БД; длинный `s3 *` — S3; ошибки без длинных дочерних spans — код. +4. Логи: Loki `{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500` + или `| event="db.unavailable"`; X-Request-Id из ответа клиента → `| request_id=""`. + +**Действия:** устранить причину по error.type (БД/S3/перегрузка); при баге — откатить версию +(`scripts/rolling-update.sh <прежняя версия>`). Resolved — когда 2-минутное окно очистится от 5xx. + +## MessengerQueueLagGrowing + +**Условие:** `max(messenger_jobs_ready_lag_seconds) > 30` и производная за 2 мин ≥ 0, `for: 1m`. +`ready_lag` — сколько ждёт worker самое старое задание, которому уже пора выполняться +(PostgreSQL `jobs`: `status='queued' AND next_attempt_at <= now()`), т. е. backlog по источнику +истины, а не по брокеру. Alert снимается, когда lag перестал расти (идёт дренаж) или < 30 с. + +**Диагностика:** +1. Панель «Saturation worker»: utilization 0 и нет `messenger_worker_capacity` → worker не работают + (`docker compose ps worker`, логи `{service_name="messenger-worker"} | event=~"worker.*"`). + Utilization ≈ 1.0 → worker живы, но не успевают (ёмкость `WORKER_REPLICAS × WORKER_CONCURRENCY`). +2. «Глубина очереди»: RabbitMQ ready растёт вместе с jobs queued → сообщения доходят, нет потребителей; + outbox pending растёт → не работает relay/брокер (см. MessengerOutboxStuck). +3. Трасса задания из окна: в трассе upload разрыв между `attachments send` (relay) и + `attachments.process.v1 process` (worker) = время в очереди; атрибут `messenger.job.queue_wait_s`. +4. `docker compose --profile tools run --rm ops inspect ` — состояние и traceparent. + +**Действия:** `./scripts/start-worker.sh` (или `docker compose up -d worker`); при насыщении — +`docker compose up -d --scale worker=N` / `WORKER_CONCURRENCY`. Проверить, что backlog убывает +(«Возраст старейшего задания» → 0) и alert resolved. Задания не теряются: они в quorum-очереди и в БД. + +## MessengerOutboxStuck + +**Условие:** старейшая созревшая неопубликованная строка outbox старше 30 с, `for: 1m`. + +**Диагностика:** `docker compose ps relay broker`; логи `{service_name="messenger-relay"} | +event=~"broker.*|db.error"`; панель «Relay: публикации outbox» (outcome=failed, error.type); +алармы памяти/диска RabbitMQ (`rabbitmq_alarms_*`). **Действия:** восстановить broker/relay; +relay опубликует накопленное само (transactional outbox), дубли гасит worker по `jobs`. + +## MessengerWorkerSaturated + +**Условие:** `sum(rate(messenger_worker_busy_time_seconds_total[2m])) / sum(messenger_worker_capacity) > 0.9`, `for: 5m`. +**Смысл:** все потоки worker заняты — новые задания копятся (предвестник MessengerQueueLagGrowing). +**Действия:** масштабировать worker; проверить «Длительность обработки job p95» и CPU worker +(«CPU saturation: доля лимита»): рост длительности при CPU у лимита → нужен CPU, а не реплики. + +## MessengerDbSlow + +**Условие:** p95 `db.client.operation.duration` API за 2 мин > 250 мс, `for: 1m` (норма — единицы мс). + +**Диагностика (отличить БД от кода):** +1. «Операции БД p95 по операции»: какая операция медленная (`get_session_user` — сессии, т. е. + почти все запросы; `add_message` — блокировка строки чата и т. п.). +2. Трасса медленного запроса: длинный CLIENT span `db <операция>` (и `db.client.connection.wait_time_ms` + в атрибутах) → время в БД. Если длинный SERVER span, а db spans короткие — проблема в коде + обработчика (пример: `handler.compute` при `X-Fault-Inject: slow-handler`, CPU scrypt `security.*`). +3. В БД: `docker compose exec -T db sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT pid, wait_event_type, wait_event, state, now()-xact_start AS xact, left(query,60) FROM pg_stat_activity WHERE datname=current_database() ORDER BY xact DESC NULLS LAST"'` + — `wait_event_type=Lock` у многих сессий → блокировка; найти держателя (самая старая транзакция). + +**Действия:** снять держателя блокировки (`SELECT pg_terminate_backend()`) или дождаться/отменить +долгую миграцию; при нехватке ресурсов — CPU/IO БД. После снятия p95 возвращается за окно 2 мин. + +## MessengerDbPoolSaturated + +**Условие:** в среднем за 1 мин > 1 запроса ждут соединение пула API, `for: 1m`. +**Смысл:** соединения заняты (медленная БД или слишком мало `DB_POOL_SIZE`); дальше — PoolTimeout (3 с) и 503. +**Действия:** как MessengerDbSlow; если БД быстрая, а пул мал — увеличить `DB_POOL_SIZE` +(и `maxconn` в HAProxy соразмерно). + +## MessengerApiLatencyHigh + +**Условие:** p95 бизнес-запросов API (все реплики, из buckets) за 5 мин > 1 с, `for: 2m`. +**Диагностика:** «p95 по маршруту» → маршрут; exemplars на панели задержки → трасса; дальше как +MessengerDbSlow (БД или код). Upload и вход дольше по природе (приём 10 MiB, scrypt). + +## MessengerTelemetryExportFailing + +**Условие:** Collector отбрасывает данные: рост `otelcol_exporter_send_failed_*` (retry исчерпан) +или `otelcol_exporter_enqueue_failed_*` (очередь экспортёра полна), `for: 1m`. +**Смысл:** телеметрия теряется (бизнес-путь — нет: SDK экспортирует асинхронно). +**Диагностика:** панель «Collector: очередь экспортёров» (size → capacity), какой exporter; +`up{job=~"loki|tempo"}`; логи Collector `docker compose logs otel-collector | grep Exporting`. +**Действия:** восстановить backend (`docker compose up -d tempo loki prometheus`); потери за период +оценить по `increase(otelcol_exporter_*_failed_*[окно])`. Восполнить потерянное нельзя — это +осознанный выбор: ограниченный буфер в памяти вместо неограниченного роста RAM. + +## MessengerTelemetryBackendDown + +**Условие:** `up{job=~"otel-collector|loki|tempo"} == 0`, `for: 1m`. +**Действия:** `docker compose ps`, `docker compose up -d `. Если упал сам Collector — +SDK приложений теряют данные после заполнения своих очередей (2048 spans / log records на +процесс); потери видны после восстановления в `otel_sdk_exporter_*_exported_total{error_type!=""}` +и `otel_sdk_processor_*_processed_total{error_type="queue_full"}`. Если упал Prometheus — alerts +не вычисляются вовсе (ограничение учебного стенда: нет внешнего мониторинга мониторинга). diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..6e469de --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,55 @@ +# Tempo (grafana/tempo:2.10.10) — хранилище ТРАСС, монолитный режим, локальный backend +# (volume tempodata: WAL + блоки), хранение 7 дней. Приём OTLP gRPC :4317 от Collector, +# запросы Grafana — HTTP :3200 (TraceQL, поиск по trace_id). +stream_over_http_enabled: true + +# Lab7: HTTP :3200 (Grafana, scrape) — TLS + обязательный client cert; приём OTLP gRPC :4317 — +# TLS + обязательный client cert (Collector). Внутренний gRPC :9096 — только 127.0.0.1. +server: + http_listen_port: 3200 + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9096 + http_tls_config: + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/internal-ca.crt + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: /run/secrets/service.crt + key_file: /run/secrets/service.key + client_ca_file: /run/secrets/internal-ca.crt + min_version: "1.2" + +ingester: + max_block_duration: 5m + trace_idle_period: 10s + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + defaults: + ingestion: + rate_limit_bytes: 15000000 + burst_size_bytes: 20000000 + max_traces_per_user: 20000 + +usage_report: + reporting_enabled: false diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..1bf8d91 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,40 @@ +-- Lab2: начальная схема. Источник истины — PostgreSQL. +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + position integer NOT NULL, -- порядок участников как в запросе + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL, + created_at timestamptz NOT NULL +); +-- Индекс под keyset-пагинацию истории: (created_at, id) внутри чата. +CREATE INDEX messages_chat_order_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, -- SHA-256 от токена; сам токен не хранится + user_id uuid NOT NULL REFERENCES users(id), + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..a46f46a --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,17 @@ +-- Lab3: общая (для всех реплик) идемпотентность POST сообщения. +-- Миграция только добавляет таблицу: код lab2 продолжает работать на новой схеме +-- (expand-only), поэтому её можно применить до замены реплик при rolling update. +-- +-- Область уникальности по контракту: пользователь + метод + путь чата + ключ. +-- Метод всегда POST, путь однозначно задаётся chat_id, поэтому ключ PK — (user_id, chat_id, idem_key). +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + idem_key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 канонического JSON тела запроса + message_id uuid NOT NULL REFERENCES messages(id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, idem_key) +); +CREATE INDEX idempotency_keys_expires_idx ON idempotency_keys (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..b408d25 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,40 @@ +-- Lab4: метаданные вложений (PostgreSQL) отдельно от байтов (приватный S3-bucket). +-- Expand-only: только новые таблицы, код lab3 продолжает работать на этой схеме. +-- +-- Жизненный цикл строки attachments: +-- pending — намерение записано ДО S3 PUT (известны размер, SHA-256, object_key); +-- ready — объект в S3 подтверждён, строка видна API и может быть прикреплена. +-- Если процесс упал/БД отказала между PUT и переводом в ready, строка остаётся pending. +-- Её находит janitor (src/janitor.py) по created_at старше grace period и удаляет +-- сначала объект в S3, затем строку. Поэтому в S3 нет объектов, о которых не знает БД. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + -- Ключ назначает сервер (случайный), filename в нём не участвует. + object_key text NOT NULL UNIQUE, + status text NOT NULL CHECK (status IN ('pending', 'ready')), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); +CREATE INDEX attachments_chat_idx ON attachments (chat_id); +-- Janitor ищет только «зависшие» pending: частичный индекс маленький. +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +-- Ссылки сообщения на вложения (порядок как в запросе). +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +-- Отпечаток тела в ключе идемпотентности уже включает attachment_ids (= [] для lab3), +-- поэтому существующие ключи остаются валидными без пересчёта. diff --git a/migrations/004_async_processing.sql b/migrations/004_async_processing.sql new file mode 100644 index 0000000..01a7956 --- /dev/null +++ b/migrations/004_async_processing.sql @@ -0,0 +1,126 @@ +-- Lab5: асинхронная обработка вложений (очередь RabbitMQ + worker). +-- +-- Жизненный цикл attachments.status: +-- pending — внутреннее: строка записана ДО PUT в staging, API её не показывает (как в lab4); +-- queued — оригинал в staging + задание + запись outbox зафиксированы ОДНОЙ транзакцией, ответ 202; +-- processing — worker взял аренду (lease) задания; +-- ready — оригинал и все варианты в конечном prefix, метаданные вариантов записаны; +-- failed — постоянная ошибка или исчерпаны попытки (error_code). +-- Источник истины по состоянию задания — таблица jobs; сообщение брокера — только «пинок» +-- с job_id: worker все решения принимает по строке jobs (FOR UPDATE). +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check + CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')); +-- Ключ staging-объекта (staging/); NULL после безопасной очистки. +ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN error_code text; +-- Версия pipeline, результат которой сейчас опубликован/ожидается (основной курс: 1). +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; + +-- Одна логическая работа = (attachment_id, pipeline_version). job_id не меняется при +-- redelivery и retry; attempt — номер управляемой попытки (растёт при каждом захвате). +CREATE TABLE jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0), + max_attempts integer NOT NULL CHECK (max_attempts >= 1), + next_attempt_at timestamptz NOT NULL, + -- Аренда: кто обрабатывает (host:boot), токен попытки (fencing) и срок. + lease_owner text, + lease_token uuid, + lease_until timestamptz, + error_code text, + last_error text, + replay_count integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + finished_at timestamptz, + UNIQUE (attachment_id, pipeline_version), + CHECK ((status = 'processing') = (lease_token IS NOT NULL)) +); +CREATE INDEX jobs_lease_idx ON jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX jobs_queued_idx ON jobs (updated_at) WHERE status = 'queued'; +CREATE INDEX jobs_failed_idx ON jobs (updated_at) WHERE status = 'failed'; + +-- Результаты: ключ эффекта (attachment_id, pipeline_version, variant) — PRIMARY KEY. +-- Повтор job не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и +-- задание. Relay публикует с publisher confirm и только потом ставит published_at. +-- available_at позволяет отложить публикацию (retry с backoff). +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs(id), + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + publish_count integer NOT NULL DEFAULT 0 +); +CREATE INDEX outbox_unpublished_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_job_idx ON outbox (job_id); + +-- Журнал переходов (audit): кто, когда, какая попытка, причина. Без секретов и bytes. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs(id), + at timestamptz NOT NULL DEFAULT clock_timestamp(), + event text NOT NULL, + attempt integer, + actor text, + detail jsonb +); +CREATE INDEX job_events_job_idx ON job_events (job_id, id); + +-- Relay ждёт NOTIFY, а не опрашивает таблицу каждые N мс. Уведомление доставляется +-- только после COMMIT; при потере уведомления relay всё равно делает опрос раз в секунду. +CREATE FUNCTION outbox_notify() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + PERFORM pg_notify('outbox', ''); + RETURN NULL; +END $$; +CREATE TRIGGER outbox_notify_trg AFTER INSERT ON outbox + FOR EACH STATEMENT EXECUTE FUNCTION outbox_notify(); + +-- Переход lab4 -> lab5: готовые вложения lab4 не имеют вариантов. Их оригинал +-- (originals/) становится staging-объектом, вложение — queued, создаётся задание. +-- Готовность заявляется только когда все варианты уровня существуют. +WITH legacy AS ( + UPDATE attachments + SET staging_key = object_key, + object_key = 'attachments/' || id || '/v1/original', + status = 'queued', + updated_at = now() + WHERE status = 'ready' + RETURNING id +), new_jobs AS ( + INSERT INTO jobs (id, attachment_id, pipeline_version, status, attempt, max_attempts, + next_attempt_at, created_at, updated_at) + SELECT gen_random_uuid(), id, 1, 'queued', 0, 5, now(), now(), now() FROM legacy + RETURNING id, attachment_id, created_at +), events AS ( + INSERT INTO job_events (job_id, event, attempt, actor, detail) + SELECT id, 'enqueued', 1, 'migration', '{"reason":"lab4 backfill"}'::jsonb FROM new_jobs +) +INSERT INTO outbox (job_id, payload, available_at) +SELECT id, + jsonb_build_object('schema_version', 1, 'job_id', id, 'type', 'attachment.process.v1', + 'attachment_id', attachment_id, 'pipeline_version', 1, 'attempt', 1, + 'created_at', to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')), + now() + FROM new_jobs; diff --git a/migrations/security/grants.sql b/migrations/security/grants.sql new file mode 100644 index 0000000..5d684da --- /dev/null +++ b/migrations/security/grants.sql @@ -0,0 +1,48 @@ +-- Lab7: минимальные права сервисных ролей PostgreSQL. Применяется src/migrate.py ПОСЛЕ миграций +-- при каждом запуске (идемпотентно: сначала REVOKE ALL, затем ровно эти GRANT). +-- Владелец объектов и DDL — только postgres (job migrate). Ни у одной сервисной роли нет +-- CREATE на схеме, TRUNCATE, REFERENCES, TRIGGER и доступа к чужим таблицам. +REVOKE ALL ON DATABASE messenger FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT CONNECT ON DATABASE messenger TO api_svc, worker_svc, relay_svc, janitor_svc, ops_svc; +GRANT USAGE ON SCHEMA public TO api_svc, worker_svc, relay_svc, janitor_svc, ops_svc; +REVOKE ALL ON ALL TABLES IN SCHEMA public FROM api_svc, worker_svc, relay_svc, janitor_svc, ops_svc; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM api_svc, worker_svc, relay_svc, janitor_svc, ops_svc; +GRANT SELECT ON schema_migrations TO api_svc, worker_svc, relay_svc, janitor_svc, ops_svc; + +-- api_svc: пользователи/сессии/чаты/сообщения/вложения; постановка задания (jobs+outbox+событие). +-- UPDATE users — только для rehash пароля при входе. +GRANT SELECT, INSERT, UPDATE ON users, sessions TO api_svc; +GRANT SELECT, INSERT ON chats, chat_members, messages, message_attachments TO api_svc; +GRANT SELECT, INSERT, DELETE ON idempotency_keys TO api_svc; +GRANT SELECT, INSERT, UPDATE, DELETE ON attachments TO api_svc; +GRANT SELECT ON attachment_variants TO api_svc; +GRANT SELECT, INSERT ON jobs, job_events, outbox TO api_svc; +-- SELECT ... FOR UPDATE OF chats (сериализация вставок в чат) требует UPDATE хотя бы на один +-- столбец. Даём только UPDATE(id): на id ссылаются FK chat_members/messages, изменить его нельзя. +GRANT UPDATE (id) ON chats TO api_svc; + +-- worker_svc: НЕТ доступа к users, sessions (хешам паролей/токенов), chats, messages. +GRANT SELECT, UPDATE ON attachments, jobs TO worker_svc; +GRANT SELECT, INSERT, UPDATE, DELETE ON attachment_variants TO worker_svc; +GRANT SELECT, INSERT ON job_events, outbox TO worker_svc; + +-- relay_svc: публикация outbox и reaper аренд. +GRANT SELECT, UPDATE, DELETE ON outbox TO relay_svc; +GRANT SELECT, UPDATE ON jobs, attachments TO relay_svc; +GRANT SELECT, INSERT ON job_events TO relay_svc; +GRANT SELECT ON attachment_variants TO relay_svc; + +-- janitor_svc: уборка брошенных вложений и staging. +GRANT SELECT, UPDATE, DELETE ON attachments TO janitor_svc; +GRANT SELECT ON jobs, outbox, attachment_variants, message_attachments TO janitor_svc; +GRANT UPDATE (id) ON jobs TO janitor_svc; -- только для SELECT ... FOR UPDATE (см. выше) +GRANT SELECT, INSERT ON job_events TO janitor_svc; + +-- ops_svc: операторский CLI (inspect/failed/replay/dlq). +GRANT SELECT, UPDATE ON jobs, attachments TO ops_svc; +GRANT SELECT ON attachment_variants TO ops_svc; +GRANT SELECT, INSERT ON job_events, outbox TO ops_svc; + +-- bigserial outbox.id, job_events.id +GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO api_svc, worker_svc, relay_svc, janitor_svc, ops_svc; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a68907d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +# Закреплённые версии. Lab5 (аудит F04): Pillow 13.0.0 (GHSA-wqgw-9xfh-qf2g, EXIF memory DoS), +# urllib3 2.8.0 (GHSA-vxq7-64xx-v4gw), minio 7.2.20 — совместимая с новым urllib3. +# Драйвер PostgreSQL для Python (из lab3). +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +# Lab4: S3-клиент (SigV4, multipart) и проверка JPEG (draft-декодирование). +minio==7.2.20 +urllib3==2.8.0 +# Lab5: Pillow также делает варианты (JPEG, WebP — libwebp входит в wheel). +Pillow==13.0.0 +# Lab5: клиент AMQP 0-9-1 для RabbitMQ (publisher confirms, manual ack, basic.qos). +pika==1.3.2 +# Lab6: OpenTelemetry SDK (traces, metrics, logs) и экспорт OTLP/HTTP protobuf в Collector. +# Транзитивные зависимости (protobuf, requests, googleapis-common-protos) зафиксированы в +# evidence/pip-freeze.txt по фактической сборке образа. +opentelemetry-api==1.45.1 +opentelemetry-sdk==1.45.1 +opentelemetry-exporter-otlp-proto-http==1.45.1 +opentelemetry-semantic-conventions==0.66b1 +# Lab7: Argon2id (libargon2 через CFFI; wheel argon2-cffi-bindings — abi3, есть для amd64/arm64). +argon2-cffi==25.1.0 +argon2-cffi-bindings==25.1.0 diff --git a/scripts/bao-init.sh b/scripts/bao-init.sh new file mode 100755 index 0000000..4d2c3ce --- /dev/null +++ b/scripts/bao-init.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# ОПЕРАТОР, один раз на новом volume baodata (повторный запуск безопасен — синхронизирует +# policies/роли/секреты): +# 1) operator init: 3 unseal shares, порог 2; root token и shares -> $OPERATOR_DIR/openbao/init.json +# (каталог оператора вне проекта, права 600). На экран ничего секретного не выводится; +# 2) unseal (scripts/bao-unseal.sh); +# 3) с root token (только в env этого процесса и `docker compose exec -e BAO_TOKEN`): +# KV v2 на kv/, policies из infra/openbao/policies, +# auth/cert с ролью на каждый сервис (allowed_dns_sans = имя сервиса, TTL токена 15 мин, +# max 1 ч), запись сервисных секретов kv/messenger/ из secrets/ (через stdin). +# Приложениям root token, unseal shares и BAO_TOKEN не передаются никогда. +set -eu +cd "$(dirname "$0")/.." +OPERATOR_DIR=${OPERATOR_DIR:-$HOME/.mtusi2026lab7-operator} +umask 077 +mkdir -p "$OPERATOR_DIR/openbao" +chmod 700 "$OPERATOR_DIR" "$OPERATOR_DIR/openbao" +INIT=$OPERATOR_DIR/openbao/init.json + +docker compose up -d openbao >/dev/null +i=0 +until docker compose exec -T openbao bao status -format=json >/dev/null 2>&1 || [ $? -eq 2 ]; do + i=$((i+1)); [ $i -gt 60 ] && { echo "openbao does not respond" >&2; exit 3; } + sleep 1 +done +initialized=$(docker compose exec -T openbao bao status -format=json 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['initialized'])" || true) +if [ "$initialized" != "True" ]; then + [ -s "$INIT" ] && { echo "$INIT exists but OpenBao is not initialized: refusing to overwrite operator material" >&2; exit 5; } + docker compose exec -T openbao bao operator init -key-shares=3 -key-threshold=2 -format=json > "$INIT.tmp" + mv "$INIT.tmp" "$INIT" + chmod 600 "$INIT" + echo "openbao initialized; root token and 3 unseal shares saved to $INIT (operator only)" +fi +./scripts/bao-unseal.sh + +BAO_TOKEN=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['root_token'])" "$INIT") +export BAO_TOKEN +bao() { docker compose exec -T -e BAO_TOKEN openbao bao "$@"; } + +bao secrets list -format=json | grep -q '"kv/"' || bao secrets enable -path=kv -version=2 kv >/dev/null +bao auth list -format=json | grep -q '"cert/"' || bao auth enable cert >/dev/null + +SERVICES="api worker relay janitor migrate s3-init ops" +for svc in $SERVICES; do + bao policy write "messenger-$svc" - < "infra/openbao/policies/messenger-$svc.hcl" >/dev/null + ttl=15m; max=1h + case $svc in migrate|s3-init) ttl=5m; max=10m ;; esac + # Роль cert auth: доверяем ТОЛЬКО internal-ca и ТОЛЬКО сертификату с SAN DNS = имя сервиса. + bao write "auth/cert/certs/$svc" display_name="$svc" token_policies="messenger-$svc" \ + certificate=@/run/secrets/internal-ca.crt allowed_dns_sans="$svc" \ + token_ttl="$ttl" token_max_ttl="$max" token_type=service >/dev/null +done + +# Сервисные секреты -> KV. JSON собирается из secrets/ и идёт в `bao kv put ... -` через stdin. +for svc in $SERVICES; do + python3 - "$svc" <<'PY' | bao kv put -mount=kv "messenger/$svc" - >/dev/null +import json, sys +svc = sys.argv[1] +r = lambda n: open("secrets/" + n).read().strip() +amqps = lambda u: "amqps://%s:%s@broker:5671/messenger" % (u, r("broker_%s_password" % u)) +s3 = lambda who: {"S3_ACCESS_KEY_ID": r("s3_%s_access_key" % who), "S3_SECRET_ACCESS_KEY": r("s3_%s_secret_key" % who)} +data = { + "api": lambda: {"POSTGRES_PASSWORD": r("db_api_password"), **s3("api")}, + "worker": lambda: {"POSTGRES_PASSWORD": r("db_worker_password"), "BROKER_URL": amqps("worker"), **s3("worker")}, + "relay": lambda: {"POSTGRES_PASSWORD": r("db_relay_password"), "BROKER_URL": amqps("relay")}, + "janitor": lambda: {"POSTGRES_PASSWORD": r("db_janitor_password"), **s3("janitor")}, + "migrate": lambda: {"POSTGRES_PASSWORD": r("postgres_password"), + **{"DB_%s_PASSWORD" % w.upper(): r("db_%s_password" % w) + for w in ("api", "worker", "relay", "janitor", "ops")}}, + "s3-init": lambda: s3("admin"), + "ops": lambda: {"POSTGRES_PASSWORD": r("db_ops_password"), "BROKER_URL": amqps("ops"), **s3("janitor")}, +}[svc]() +json.dump(data, sys.stdout) +PY +done +unset BAO_TOKEN +echo "openbao configured: kv v2 'kv/', audit file->stdout, auth/cert roles: $SERVICES" diff --git a/scripts/bao-unseal.sh b/scripts/bao-unseal.sh new file mode 100755 index 0000000..cb1539c --- /dev/null +++ b/scripts/bao-unseal.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# ОПЕРАТОР: unseal OpenBao после старта/рестарта контейнера (file storage -> процесс sealed). +# Берёт ровно threshold (2 из 3) unseal shares из каталога оператора $OPERATOR_DIR/openbao +# (вне проекта, 600, никогда не монтируется в контейнеры) и передаёт их через stdin — значения не +# появляются ни в выводе, ни в аргументах процессов. Повторный вызов на unsealed OpenBao — no-op. +set -eu +cd "$(dirname "$0")/.." +OPERATOR_DIR=${OPERATOR_DIR:-$HOME/.mtusi2026lab7-operator} +INIT=$OPERATOR_DIR/openbao/init.json +[ -s "$INIT" ] || { echo "no $INIT: run scripts/bao-init.sh first" >&2; exit 2; } +# Ждём, пока процесс OpenBao начнёт отвечать (после рестарта контейнера). +i=0 +until docker compose exec -T openbao bao status -format=json >/dev/null 2>&1 || [ $? -eq 2 ]; do + i=$((i+1)); [ $i -gt 60 ] && { echo "openbao does not respond" >&2; exit 3; } + sleep 1 +done +sealed() { docker compose exec -T openbao bao status -format=json 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['sealed'])"; } +if [ "$(sealed)" = "False" ]; then + echo "openbao already unsealed"; exit 0 +fi +for idx in 0 1; do + python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['unseal_keys_b64'][int(sys.argv[2])])" "$INIT" "$idx" \ + | docker compose exec -T openbao bao write -format=json sys/unseal key=- >/dev/null +done +[ "$(sealed)" = "False" ] && echo "openbao unsealed (2 of 3 shares)" || { echo "unseal failed" >&2; exit 4; } diff --git a/scripts/build_dashboard.py b/scripts/build_dashboard.py new file mode 100755 index 0000000..c5c19df --- /dev/null +++ b/scripts/build_dashboard.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Генератор импортируемого dashboard Grafana: infra/grafana/dashboards/messenger.json. + +JSON лежит в Git и подхватывается provisioning-ом; этот скрипт — воспроизводимый способ его +изменить (python3 scripts/build_dashboard.py). Только стандартная библиотека. +Все запросы ссылаются на реальные имена метрик (см. REPORT.md «Схема сигналов»). +Перцентили — histogram_quantile по sum by (le) buckets всех реплик, не среднее p95. +""" +import json +from pathlib import Path + +OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json" +PROM = {"type": "prometheus", "uid": "prometheus"} +LOKI = {"type": "loki", "uid": "loki"} +TEMPO = {"type": "tempo", "uid": "tempo"} +API = 'job="messenger-api", instance=~"$instance"' +BUSINESS = API + ', http_route!~"/health/.*|unmatched"' + +_panels = [] +_id = [0] +_pos = {"x": 0, "y": 0, "h": 0} + + +def _next_id(): + _id[0] += 1 + return _id[0] + + +def place(w, h): + if _pos["x"] + w > 24: + _pos["x"] = 0 + _pos["y"] += _pos["h"] + _pos["h"] = 0 + g = {"x": _pos["x"], "y": _pos["y"], "w": w, "h": h} + _pos["x"] += w + _pos["h"] = max(_pos["h"], h) + return g + + +def row(title): + _pos["x"] = 0 + _pos["y"] += _pos["h"] + _pos["h"] = 0 + _panels.append({"type": "row", "id": _next_id(), "title": title, "collapsed": False, + "gridPos": {"x": 0, "y": _pos["y"], "w": 24, "h": 1}, "panels": []}) + _pos["y"] += 1 + + +def target(expr, legend="", ref="A", exemplar=False, instant=False): + t = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": ref, + "range": not instant, "instant": instant} + if exemplar: + t["exemplar"] = True + return t + + +def ts(title, targets, unit="short", w=8, h=8, description="", stack=False, min0=True, thresholds=None): + refs = "ABCDEFGHIJ" + tg = [target(*t[:2], ref=refs[i], exemplar=(len(t) > 2 and t[2])) for i, t in enumerate(targets)] + p = {"type": "timeseries", "id": _next_id(), "title": title, "description": description, + "datasource": PROM, "gridPos": place(w, h), "targets": tg, + "fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, + "showPoints": "never", + "stacking": {"mode": "normal" if stack else "none"}}, + **({"min": 0} if min0 else {})}, + "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, + "tooltip": {"mode": "multi", "sort": "desc"}}} + if thresholds: + p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds} + p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line+area"} + _panels.append(p) + + +def stat(title, expr, unit="short", w=3, h=4, steps=None, description="", decimals=None): + p = {"type": "stat", "id": _next_id(), "title": title, "description": description, "datasource": PROM, + "gridPos": place(w, h), "targets": [target(expr, "", instant=True)], + "fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps or [ + {"color": "green", "value": None}]}}, "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "colorMode": "background", "graphMode": "area", "textMode": "value"}} + if decimals is not None: + p["fieldConfig"]["defaults"]["decimals"] = decimals + _panels.append(p) + + +def build(): + # ------------------------------------------------------------------ обзор + row("Обзор и SLO") + stat("RPS бизнес-запросов", "messenger:http_requests:rate2m", "reqps", + description="sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))") + stat("5xx, доля (2m)", "messenger:http_error_ratio:rate2m", "percentunit", decimals=2, + steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}], + description="Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).") + stat("p95 API (5m)", "messenger:http_latency_seconds:p95_5m", "s", decimals=3, + steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.5}, {"color": "red", "value": 1}]) + stat("SLO доступности, 7d", "messenger:slo_availability:ratio_7d", "percentunit", decimals=3, + steps=[{"color": "red", "value": None}, {"color": "green", "value": 0.995}], + description="SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.") + stat("Остаток error budget, 7d", "messenger:slo_availability:error_budget_remaining_7d", "percentunit", + decimals=1, steps=[{"color": "red", "value": None}, {"color": "orange", "value": 0.25}, + {"color": "green", "value": 0.5}]) + stat("Time to ready p95", "messenger:time_to_ready_seconds:p95_5m", "s", decimals=2, + steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}], + description="От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.") + stat("Ready lag очереди", "max(messenger_jobs_ready_lag_seconds) or vector(0)", "s", decimals=0, + steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}], + description="Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).") + stat("Alerts firing", 'count(ALERTS{alertstate="firing"}) or vector(0)', "short", + steps=[{"color": "green", "value": None}, {"color": "red", "value": 1}]) + _panels.append({ + "type": "state-timeline", "id": _next_id(), "title": "История alerts (pending/firing -> resolved)", + "datasource": PROM, "gridPos": place(24, 6), + "targets": [target('max by (alertname, source, alertstate) (ALERTS)', "{{alertname}} {{source}} {{alertstate}}")], + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, + "thresholds": {"mode": "absolute", "steps": [ + {"color": "transparent", "value": None}, {"color": "red", "value": 1}]}}, + "overrides": []}, + "options": {"showValue": "never", "mergeValues": True, "rowHeight": 0.8, + "legend": {"showLegend": False}}}) + + # ------------------------------------------------------------------ API + row("API: трафик, ошибки, задержка") + ts("RPS по маршруту", [("sum by (http_route) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % API, + "{{http_route}}")], "reqps", stack=True) + ts("Ответы по классу статуса", [( + "sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % BUSINESS, + "{{http_response_status_class}}")], "reqps", stack=True) + ts("Доля 5xx: API и LB", [ + ("messenger:http_error_ratio:rate2m", "API (без /health)"), + ("messenger:lb_error_ratio:rate2m", "HAProxy fe_api")], "percentunit", + thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 0.05}], + description="Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.") + ts("Задержка API p50/p95/p99 (из histogram)", [ + ("histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p50", True), + ("histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p95", True), + ("histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p99", True)], + "s", w=12, description="Точки-exemplars ведут в трассу Tempo конкретного запроса.") + ts("p95 по маршруту", [( + "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{%s}[5m])))" % BUSINESS, + "{{http_route}}")], "s", w=12) + ts("5xx по причине (error.type)", [( + 'sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{%s, http_response_status_class="5xx"}[1m]))' % API, + "{{error_type}} {{http_route}}")], "reqps", stack=True) + ts("Запросы в обработке (saturation API)", [ + ("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], "short") + ts("HAProxy: UP-реплики и очередь LB", [ + ('haproxy_backend_active_servers{proxy="be_api"}', "UP реплик"), + ('haproxy_backend_current_queue{proxy="be_api"}', "очередь LB")], "short") + + # ------------------------------------------------------------------ worker + row("Worker и очередь заданий") + ts("Задания по исходу (в минуту)", [( + "sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", "{{messenger_job_outcome}}")], + "short", stack=True, description="ready / retry / failed / duplicate / lease_lost / rejected / requeued") + ts("Длительность обработки job p50/p95/p99", [ + ('histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p50", True), + ('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p95", True), + ('histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p99", True)], + "s") + ts("Time to ready p50/p95/p99", [ + ('histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p50", True), + ('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p95", True), + ('histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p99", True)], + "s", description="Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.") + ts("Глубина очереди: PostgreSQL и RabbitMQ", [ + ('max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~"queued|processing"})', "jobs {{messenger_job_status}} (БД)"), + ('max(messenger_outbox_pending)', "outbox не опубликовано"), + ('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', "RabbitMQ ready"), + ('sum(rabbitmq_detailed_queue_messages_unacked{queue="attachments.process.v1"})', "RabbitMQ unacked"), + ('sum(rabbitmq_detailed_queue_messages{queue="attachments.dlq"})', "DLQ")], "short") + ts("Возраст старейшего задания и lag", [ + ('max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)', "oldest {{messenger_job_status}}"), + ("max(messenger_jobs_ready_lag_seconds)", "ready lag"), + ("max(messenger_outbox_oldest_age_seconds)", "outbox oldest")], "s", + thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 30}]) + ts("Ожидание в очереди p95 (до claim)", [ + ("histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", "p95 queue wait"), + ("histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", "p95 outbox -> broker")], + "s", description="Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).") + ts("Saturation worker: занятость потребителей", [ + ("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", "utilization (busy_time/capacity)"), + ("sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", "busy/capacity (мгновенно)")], + "percentunit", description="1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.") + ts("Relay: публикации outbox", [ + ("sum by (outcome) (rate(messenger_outbox_published_total[1m]))", "{{outcome}}")], "short") + + # ------------------------------------------------------------------ DB / S3 + row("PostgreSQL и S3") + ts("Пул соединений API: used / idle / max", [ + ('sum by (instance, db_client_connection_state) (db_client_connection_count{%s})' % API, + "{{instance}} {{db_client_connection_state}}"), + ('max(db_client_connection_max{%s})' % API, "max на реплику")], "short") + ts("Ожидающие соединение и таймауты пула", [ + ('sum by (instance) (db_client_connection_pending_requests{%s})' % API, "pending {{instance}}"), + ('sum by (instance) (increase(db_client_connection_timeouts_total{%s}[1m]))' % API, "timeouts/мин {{instance}}")], + "short") + ts("Ожидание соединения пула p95", [ + ('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{%s}[1m])))' % API, "p95 wait")], + "s") + ts("Операции БД p95 по операции (API)", [ + ('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{%s}[1m])))' % API, + "{{db_operation_name}}")], "s", w=12) + ts("Операции БД worker/relay p95", [ + ('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~"messenger-worker|messenger-relay"}[5m])))', + "{{db_operation_name}}")], "s", w=12) + ts("Ошибки операций БД", [ + ('sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=""}[1m]))', + "{{job}} {{db_operation_name}} {{error_type}}")], "short") + ts("S3 p95 по операции", [ + ('histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))', + "{{messenger_s3_operation}}")], "s") + ts("S3 ошибки", [ + ('sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!="ok"}[1m]))', + "{{job}} {{messenger_s3_operation}} {{outcome}}")], "short") + + # ------------------------------------------------------------------ ресурсы + row("Ресурсы контейнеров (CPU/RAM)") + ts("CPU, ядер (cgroup контейнера)", [ + ("sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", "{{job}} {{instance}}")], "short", w=12) + ts("CPU saturation: доля лимита", [ + ("rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", "{{job}} {{instance}}")], + "percentunit", w=12) + ts("Память контейнеров приложений", [ + ("container_memory_usage_bytes", "{{job}} {{instance}}")], "bytes", w=12) + ts("Память: доля лимита", [ + ("container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", "{{job}} {{instance}}")], + "percentunit", w=12) + ts("Память telemetry-стека (RSS)", [ + ('process_resident_memory_bytes{job=~"prometheus|loki|tempo"}', "{{job}}"), + ('{__name__=~"otelcol_process_memory_rss(_bytes)?"}', "otel-collector RSS (вкл. страницы бинарника)"), + ('otelcol_process_runtime_heap_alloc_bytes', "otel-collector Go heap"), + ('rabbitmq_process_resident_memory_bytes', "rabbitmq")], "bytes", w=12) + ts("CPU telemetry-стека", [ + ('rate(process_cpu_seconds_total{job=~"prometheus|loki|tempo"}[1m])', "{{job}}"), + ('rate({__name__=~"otelcol_process_cpu_seconds(_total)?"}[1m])', "otel-collector")], "short", w=12) + + # ------------------------------------------------------------------ telemetry + row("Конвейер телеметрии (потери)") + ts("Collector: принято / отправлено / потеряно (spans)", [ + ('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[1m]))', "принято"), + ('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[1m]))', "отправлено в Tempo"), + ('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m]))', "ошибка отправки"), + ('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[1m]))', "очередь полна (потеря)"), + ('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[1m]))', "отказ приёма (memory_limiter)")], "short") + ts("Collector: логи", [ + ('sum(rate({__name__=~"otelcol_receiver_accepted_log_records(_total)?"}[1m]))', "принято"), + ('sum(rate({__name__=~"otelcol_exporter_sent_log_records(_total)?"}[1m]))', "отправлено в Loki"), + ('sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m]))', "ошибка отправки"), + ('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[1m]))', "очередь полна (потеря)")], "short") + ts("Collector: очередь экспортёров", [ + ("max by (exporter) (otelcol_exporter_queue_size)", "size {{exporter}}"), + ("max by (exporter) (otelcol_exporter_queue_capacity)", "capacity {{exporter}}")], "short") + ts("SDK приложений: потери на стороне процесса", [ + ('sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[1m]))', "spans dropped {{job}} {{error_type}}"), + ('sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=""}[1m]))', "logs dropped {{job}} {{error_type}}"), + ('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[1m]))', "span export failed {{job}} {{error_type}}")], + "short", w=12) + ts("SDK: заполнение очереди BatchSpanProcessor", [ + ("max by (job, instance) (otel_sdk_processor_span_queue_size)", "{{job}} {{instance}}"), + ("max(otel_sdk_processor_span_queue_capacity)", "capacity")], "short", w=12) + + # ------------------------------------------------------------------ logs / traces + row("Логи и трассы") + _panels.append({ + "type": "logs", "id": _next_id(), "title": "Логи WARN/ERROR приложений (Loki)", "datasource": LOKI, + "gridPos": place(24, 10), + "targets": [{"datasource": LOKI, "refId": "A", "queryType": "range", + "expr": '{service_name=~"messenger-.+"} | severity_text=~"WARN.*|ERROR.*|CRITICAL"'}], + "options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending", + "dedupStrategy": "none", "prettifyLogMessage": False}}) + _panels.append({ + "type": "logs", "id": _next_id(), "title": "HTTP 5xx (event=http.request)", "datasource": LOKI, + "gridPos": place(12, 10), + "targets": [{"datasource": LOKI, "refId": "A", "queryType": "range", + "expr": '{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500'}], + "options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}}) + _panels.append({ + "type": "table", "id": _next_id(), "title": "Последние трассы обработки заданий (Tempo)", "datasource": TEMPO, + "gridPos": place(12, 10), + "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces", + "query": '{ resource.service.name = "messenger-worker" && name = "attachments.process.v1 process" }'}]}) + _panels.append({ + "type": "table", "id": _next_id(), "title": "Медленные запросы API > 1 s (Tempo)", "datasource": TEMPO, + "gridPos": place(12, 10), + "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces", + "query": '{ resource.service.name = "messenger-api" && kind = server && duration > 1s }'}]}) + _panels.append({ + "type": "table", "id": _next_id(), "title": "Трассы с ошибкой (Tempo)", "datasource": TEMPO, + "gridPos": place(12, 10), + "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces", + "query": '{ status = error }'}]}) + + return { + "uid": "messenger-overview", + "title": "Messenger — наблюдаемость (lab6)", + "description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.", + "tags": ["messenger", "lab6", "otel"], + "timezone": "browser", + "editable": False, + "graphTooltip": 1, + "refresh": "10s", + "schemaVersion": 39, + "version": 1, + "time": {"from": "now-30m", "to": "now"}, + "templating": {"list": [{ + "name": "instance", "label": "API-реплика", "type": "query", "datasource": PROM, + "query": {"query": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)', + "refId": "PrometheusVariableQueryEditor-VariableQuery"}, + "definition": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)', + "includeAll": True, "multi": True, "allValue": ".*", "current": {"text": "All", "value": "$__all"}, + "refresh": 2, "sort": 1}]}, + "annotations": {"list": [{ + "builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]}, + "panels": _panels, + } + + +if __name__ == "__main__": + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + print("written %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(_panels))) diff --git a/scripts/check_rules.sh b/scripts/check_rules.sh new file mode 100755 index 0000000..3f222e9 --- /dev/null +++ b/scripts/check_rules.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Офлайн-проверка конфигураций телеметрии теми же образами, что и стенд (сеть не нужна): +# promtool check config + check rules + unit-тесты alert rules, +# otelcol-contrib validate, loki -verify-config. +# Использование: scripts/check_rules.sh (код возврата 0 — всё валидно) +set -eu +cd "$(dirname "$0")/.." +img() { grep -E "^\s+image: $1" compose.yaml | head -1 | awk '{print $2}'; } +PROM=$(img prom/prometheus) +OTEL=$(img otel/opentelemetry-collector-contrib) +LOKI=$(img grafana/loki) +echo "== promtool check config ($PROM)" +# Lab7: конфиги ссылаются на TLS-файлы /run/secrets/* — монтируем сертификаты стенда (secrets/pki). +pki() { # pki -> -v аргументы для CA и leaf + echo "-v $PWD/secrets/pki/internal-ca.crt:/run/secrets/internal-ca.crt:ro -v $PWD/secrets/pki/$1.crt:/run/secrets/service.crt:ro -v $PWD/secrets/pki/$1.key:/run/secrets/service.key:ro" +} +docker run --rm $(pki prometheus) -v "$PWD/infra/prometheus:/etc/prometheus:ro" --entrypoint promtool "$PROM" check config /etc/prometheus/prometheus.yml +echo "== promtool check web-config (TLS + обязательный client cert)" +docker run --rm $(pki prometheus) -v "$PWD/infra/prometheus:/etc/prometheus:ro" --entrypoint promtool "$PROM" check web-config /etc/prometheus/web.yml +echo "== promtool test rules" +docker run --rm -v "$PWD/infra/prometheus:/w:ro" -w /w/tests --entrypoint promtool "$PROM" test rules messenger.test.yml +echo "== otelcol validate ($OTEL)" +docker run --rm $(pki otel-collector) -v "$PWD/infra/otel/collector.yaml:/c.yaml:ro" "$OTEL" validate --config=/c.yaml && echo "collector config OK" +echo "== loki -verify-config ($LOKI)" +docker run --rm $(pki loki) -v "$PWD/infra/loki/loki.yaml:/c.yaml:ro" "$LOKI" -config.file=/c.yaml -verify-config && echo "loki config OK" +echo "== dashboard JSON is up to date with scripts/build_dashboard.py" +tmp=$(mktemp) +cp infra/grafana/dashboards/messenger.json "$tmp" +python3 scripts/build_dashboard.py >/dev/null +cmp -s "$tmp" infra/grafana/dashboards/messenger.json && echo "dashboard OK" || { echo "dashboard JSON differs from generator" >&2; exit 1; } diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 0000000..da6a6c3 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,135 @@ +#!/bin/sh +# Lab7: локальные секреты, PKI и .env учебного стенда. Повторный запуск ничего не перезаписывает, +# значения не печатаются. Ничего из созданного не попадает в Git (.gitignore: secrets/, *.key, *.pem) +# и в образы (.dockerignore). +# +# Где что лежит: +# $OPERATOR_DIR (по умолчанию ~/.mtusi2026lab7-operator, вне проекта, 700): +# ca/*.key — закрытые ключи public-ca и internal-ca (не монтируются ни в один контейнер); +# openbao/ — root token и unseal shares (создаёт scripts/bao-init.sh). +# secrets/ (в проекте, вне Git, 700): +# pki/*.crt|*.key — leaf-сертификаты сервисов (каждый контейнер получает ТОЛЬКО свой ключ); +# pki/test/ — rogue CA и identity intruder для отрицательных тестов; +# *_password, s3_* — исходные значения сервисных credentials. Приложения (api, worker, +# relay, janitor, migrate, s3-init, ops) получают их НЕ из этих файлов, +# а из OpenBao (scripts/bao-init.sh кладёт их в KV). Файлы нужны серверной +# стороне: s3_identities.json (SeaweedFS), rabbitmq_definitions.json (только +# хеши паролей), postgres_password (суперпользователь в db). +# +# S3 (SeaweedFS) — разные ключи и права (secrets/s3_identities.json): +# admin — только job s3-init; api — Read+Write; worker — Read+Write; janitor — List+Write. +# RabbitMQ — пользователи с минимальными правами в vhost messenger: +# relay — только publish в exchange attachments; worker — только consume из attachments.process.v1; +# ops — consume из DLQ и publish в attachments. Ни у кого нет configure. +# PostgreSQL — отдельные роли api_svc, worker_svc, relay_svc, janitor_svc, ops_svc с грантами по +# таблицам (migrations/grants.sql); суперпользователь postgres — только у job migrate. +set -eu +cd "$(dirname "$0")/.." +umask 077 +OPERATOR_DIR=${OPERATOR_DIR:-$HOME/.mtusi2026lab7-operator} +mkdir -p secrets/pki "$OPERATOR_DIR" +chmod 700 secrets "$OPERATOR_DIR" +gen() { # gen + if [ ! -s "secrets/$1" ]; then + python3 -c "import secrets,sys; print(secrets.token_urlsafe(int(sys.argv[1])))" "$2" > "secrets/$1" + echo "created secrets/$1" + fi +} +gen postgres_password 24 +for who in api worker relay janitor ops; do + gen "db_${who}_password" 24 +done +gen grafana_admin_password 18 +for who in admin api worker janitor; do + if [ ! -s "secrets/s3_${who}_access_key" ]; then + python3 -c "import secrets; print('MTUSI' + secrets.token_hex(8).upper())" > "secrets/s3_${who}_access_key" + echo "created secrets/s3_${who}_access_key" + fi + gen "s3_${who}_secret_key" 30 +done +for who in relay worker ops; do + gen "broker_${who}_password" 24 +done + +# PKI: OpenSSL закреплённого образа (тот же, на котором собрано приложение). +PKI_IMAGE=python:3.12.7-slim@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d +docker run --rm --network none --user "$(id -u):$(id -g)" \ + -e PKI_CA_DAYS -e PKI_LEAF_DAYS \ + -v "$PWD/secrets/pki:/out" -v "$OPERATOR_DIR:/op" -v "$PWD/scripts/pki-issue.sh:/pki-issue.sh:ro" \ + "$PKI_IMAGE" sh /pki-issue.sh + +BUCKET=$(grep -E '^S3_BUCKET=' .env 2>/dev/null | cut -d= -f2 || true) +BUCKET=${BUCKET:-messenger-attachments} +python3 - "$BUCKET" <<'PY' +import base64, hashlib, json, os, sys +b = sys.argv[1] +def read(name): + return open("secrets/" + name).read().strip() +def ident(name, actions): + return {"name": name, "credentials": [{"accessKey": read("s3_%s_access_key" % name), + "secretKey": read("s3_%s_secret_key" % name)}], "actions": actions} +cfg = {"identities": [ + ident("admin", ["Admin", "Read", "List", "Tagging", "Write"]), + ident("api", ["Read:" + b, "Write:" + b]), + ident("worker", ["Read:" + b, "Write:" + b]), + ident("janitor", ["List:" + b, "Write:" + b]), +]} +with open("secrets/s3_identities.json", "w") as f: + json.dump(cfg, f, indent=1) + +# RabbitMQ: rabbit_password_hashing_sha256 = base64(salt4 + sha256(salt4 + password)). +def rmq_hash(password): + salt = os.urandom(4) + return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode() +vhost = "messenger" +users, perms = [], [] +rights = { + "relay": {"configure": "^$", "write": "^attachments$", "read": "^$"}, + "worker": {"configure": "^$", "write": "^$", "read": r"^attachments\.process\.v1$"}, + "ops": {"configure": "^$", "write": "^attachments$", "read": r"^attachments\.dlq$"}, +} +for user, r in rights.items(): + users.append({"name": user, "password_hash": rmq_hash(read("broker_%s_password" % user)), + "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []}) + perms.append(dict(user=user, vhost=vhost, **r)) +x = lambda name, kind: {"name": name, "vhost": vhost, "type": kind, "durable": True, "auto_delete": False, + "internal": False, "arguments": {}} +definitions = { + "users": users, + "vhosts": [{"name": vhost}], + "permissions": perms, + "exchanges": [x("attachments", "direct"), x("attachments.dlx", "fanout")], + "queues": [ + {"name": "attachments.process.v1", "vhost": vhost, "durable": True, "auto_delete": False, + "arguments": {"x-queue-type": "quorum", "x-delivery-limit": 10, + "x-dead-letter-exchange": "attachments.dlx", + "x-dead-letter-strategy": "at-least-once", "x-overflow": "reject-publish", + "x-max-length": 100000}}, + {"name": "attachments.dlq", "vhost": vhost, "durable": True, "auto_delete": False, + "arguments": {"x-queue-type": "quorum"}}, + ], + "bindings": [ + {"source": "attachments", "vhost": vhost, "destination": "attachments.process.v1", + "destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}}, + {"source": "attachments.dlx", "vhost": vhost, "destination": "attachments.dlq", + "destination_type": "queue", "routing_key": "", "arguments": {}}, + ], +} +if not os.path.exists("secrets/rabbitmq_definitions.json"): + with open("secrets/rabbitmq_definitions.json", "w") as f: + json.dump(definitions, f, indent=1) + print("created secrets/rabbitmq_definitions.json") +PY +# Файлы монтируются в контейнеры как Docker secrets (bind mount): uid процесса внутри контейнера +# (70 postgres, 10001 app, 100 rabbitmq/openbao, 99 haproxy, 472 grafana, 65534 prometheus, 10001 loki/tempo) +# должен их читать. От других пользователей ХОСТА их закрывает каталог secrets/ (700). Каждый +# контейнер получает только свои файлы (см. compose.yaml: secrets per service). +chmod 644 secrets/*_password secrets/s3_* secrets/rabbitmq_definitions.json secrets/pki/*.crt secrets/pki/*.key \ + secrets/pki/*.pem secrets/pki/test/* +chmod 700 secrets/pki secrets/pki/test +if [ ! -f .env ]; then + cp .env.example .env + chmod 600 .env + echo "created .env from .env.example" +fi +echo "operator material: $OPERATOR_DIR (CA keys; OpenBao init material after scripts/bao-init.sh)" diff --git a/scripts/pki-issue.sh b/scripts/pki-issue.sh new file mode 100755 index 0000000..8b93a49 --- /dev/null +++ b/scripts/pki-issue.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# Выпуск учебного PKI lab7. Запускается init-secrets.sh ВНУТРИ закреплённого образа +# python:3.12.7-slim (OpenSSL 3.0) — результат не зависит от openssl/LibreSSL хоста. +# /op — каталог оператора (вне проекта): закрытые ключи CA, никогда не монтируется в стенд; +# /out — secrets/pki проекта (вне Git): сертификаты CA и leaf-сертификаты/ключи сервисов. +# Повторный запуск ничего не перевыпускает (кроме явно удалённых файлов); значения не печатаются. +# +# Два CA с разным назначением: +# public-ca — внешний TLS (HAProxy :8443, Grafana :3000). Его сертификат раздаётся клиентам. +# internal-ca — mTLS/TLS внутри стенда (LB->API, сервисы -> PostgreSQL/S3/RabbitMQ/Collector/ +# Prometheus/Loki/Tempo/OpenBao). Клиенты извне ему не доверяют. +# Третий, rogue-ca, — ТОЛЬКО для отрицательных тестов (сертификат «lb» от чужого CA). +set -eu +umask 077 +OP=/op +OUT=/out +DAYS_CA=${PKI_CA_DAYS:-365} +DAYS_LEAF=${PKI_LEAF_DAYS:-30} +mkdir -p "$OP/ca" "$OUT/test" + +ca() { # ca + name=$1; cn=$2; keydir=$3 + if [ ! -s "$keydir/$name.key" ] || [ ! -s "$OUT/$name.crt" ]; then + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$keydir/$name.key" 2>/dev/null + openssl req -x509 -new -key "$keydir/$name.key" -sha256 -days "$DAYS_CA" \ + -subj "/O=MTUSI lab7/CN=$cn" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -addext "subjectKeyIdentifier=hash" \ + -out "$OUT/$name.crt" + echo "created CA $name (key in operator dir)" + fi +} + +leaf() { # leaf + caname=$1; name=$2; eku=$3; shift 3 + dir=$OUT; [ "$caname" = rogue-ca ] && dir=$OUT/test + [ "$name" = intruder ] && dir=$OUT/test + if [ -s "$dir/$name.crt" ] && [ -s "$dir/$name.key" ]; then return 0; fi + cakey=$OP/ca/$caname.key; [ "$caname" = rogue-ca ] && cakey=$OUT/test/rogue-ca.key + cacrt=$OUT/$caname.crt; [ "$caname" = rogue-ca ] && cacrt=$OUT/test/rogue-ca.crt + san="" + for s in "$@"; do san="${san:+$san,}$s"; done + ext=$(mktemp) + cat > "$ext" </dev/null + openssl req -new -key "$dir/$name.key" -subj "/O=MTUSI lab7/CN=$name" -out "$dir/$name.csr" + openssl x509 -req -in "$dir/$name.csr" -CA "$cacrt" -CAkey "$cakey" -CAcreateserial \ + -CAserial "$OP/ca/$caname.srl" -days "$DAYS_LEAF" -sha256 -extfile "$ext" -out "$dir/$name.crt" 2>/dev/null + rm -f "$dir/$name.csr" "$ext" + echo "issued $name ($caname, $DAYS_LEAF days)" +} + +ca public-ca "MTUSI lab7 Public Ingress CA" "$OP/ca" +ca internal-ca "MTUSI lab7 Internal Services CA" "$OP/ca" +# rogue CA — тестовый, его ключ не секретен для стенда (лежит в secrets/pki/test). +if [ ! -s "$OUT/test/rogue-ca.key" ]; then + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$OUT/test/rogue-ca.key" 2>/dev/null + openssl req -x509 -new -key "$OUT/test/rogue-ca.key" -sha256 -days 30 -subj "/O=Attacker/CN=Rogue CA" \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign" \ + -out "$OUT/test/rogue-ca.crt" + cp "$OP/ca/internal-ca.srl" "$OP/ca/rogue-ca.srl" 2>/dev/null || true +fi + +BOTH=serverAuth,clientAuth +# Внешние (public CA): клиенты проверяют hostname localhost / 127.0.0.1. +leaf public-ca lb-public serverAuth DNS:localhost IP:127.0.0.1 +leaf public-ca grafana-public serverAuth DNS:localhost IP:127.0.0.1 +# Внутренние identities (internal CA). SAN DNS = имя сервиса Compose = проверяемая identity. +# lb — клиентская identity LB для mTLS к API (и серверная для scrape :8404). +leaf internal-ca lb "$BOTH" DNS:lb +leaf internal-ca api "$BOTH" DNS:api +for s in worker relay janitor migrate s3-init ops grafana; do + leaf internal-ca "$s" clientAuth "DNS:$s" +done +leaf internal-ca db serverAuth DNS:db DNS:localhost +for s in s3 broker otel-collector prometheus loki tempo openbao; do + leaf internal-ca "$s" "$BOTH" "DNS:$s" DNS:localhost IP:127.0.0.1 +done +# Отрицательные тесты: доверенный CA, но запрещённая identity; чужой CA с «правильным» SAN lb. +leaf internal-ca intruder "$BOTH" DNS:intruder +leaf rogue-ca rogue-lb "$BOTH" DNS:lb + +# HAProxy читает сертификат и ключ одним PEM. +cat "$OUT/lb-public.crt" "$OUT/lb-public.key" > "$OUT/lb-public.pem" +cat "$OUT/lb.crt" "$OUT/lb.key" > "$OUT/lb.pem" +chmod 600 "$OP/ca/"* diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..84ecb0e --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Перезапуск ВСЕХ хранилищ — данных (PostgreSQL, SeaweedFS S3, RabbitMQ), секретов (OpenBao) и +# телеметрии (Prometheus, Loki, Tempo) — и всех процессов (API-реплики, relay, worker, janitor, LB, +# Collector, Grafana) с сохранением volumes. Не выполняет down -v, не пересоздаёт БД/bucket/очереди. +# +# Lab7: OpenBao после рестарта sealed — это хук ОПЕРАТОРА, поэтому он выполняет unseal +# (scripts/bao-unseal.sh, 2 из 3 shares из $OPERATOR_DIR вне проекта). Процессы приложения +# заново проходят cert auth и читают свои секреты; старые токены не нужны. +set -eu +cd "$(dirname "$0")/.." +docker compose restart openbao db s3 broker prometheus loki tempo +./scripts/bao-unseal.sh >/dev/null +docker compose up -d --no-deps --no-recreate --wait openbao db s3 broker prometheus +docker compose restart otel-collector api relay worker janitor lb grafana diff --git a/scripts/scan_secrets.py b/scripts/scan_secrets.py new file mode 100755 index 0000000..f3845ab --- /dev/null +++ b/scripts/scan_secrets.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Проверка перед commit (lab7): значения секретов не встречаются ни в одном файле репозитория +вне secrets/ и .env. Значения не печатаются — только имена файлов. + +Источники значений: + * secrets/**: пароли, ключи S3, password_hash RabbitMQ, ЗАКРЫТЫЕ КЛЮЧИ leaf (тело PEM по строкам); + * $OPERATOR_DIR (по умолчанию ~/.mtusi2026lab7-operator): закрытые ключи CA, root token и + unseal shares OpenBao (init.json). +Публичные сертификаты (*.crt) секретом не являются и не проверяются. + python3 scripts/scan_secrets.py (код 0 — чисто)""" +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OPERATOR_DIR = Path(os.environ.get("OPERATOR_DIR", Path.home() / ".mtusi2026lab7-operator")) +SKIP_DIRS = {".git", "secrets", ".venv", "__pycache__", "node_modules"} +values = set() + + +def add_pem(text): + for line in text.splitlines(): + line = line.strip() + if len(line) >= 16 and not line.startswith("-----"): + values.add(line) + + +def walk(x): + if isinstance(x, dict): + for k, v in x.items(): + if k in ("accessKey", "secretKey", "password_hash", "root_token", "unseal_keys_b64", "unseal_keys_hex"): + for item in (v if isinstance(v, list) else [v]): + values.add(str(item)) + walk(v) + elif isinstance(x, list): + for v in x: + walk(v) + + +sources = [f for f in (ROOT / "secrets").rglob("*") if f.is_file() and f.suffix != ".crt"] +sources += [f for f in OPERATOR_DIR.rglob("*") if f.is_file()] if OPERATOR_DIR.is_dir() else [] +for f in sources: + text = f.read_text(encoding="utf-8", errors="replace").strip() + if f.suffix == ".json": + walk(json.loads(text)) + elif "PRIVATE KEY" in text: + add_pem(text) + elif len(text) >= 8 and "\n" not in text: + values.add(text) + if "://" in text and "@" in text: + values.add(text.split("://", 1)[1].split("@", 1)[0].split(":", 1)[-1]) +hits = [] +for p in ROOT.rglob("*"): + if p.is_dir() or set(p.relative_to(ROOT).parts) & SKIP_DIRS or p.name == ".env": + continue + try: + data = p.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + if any(v in data for v in values): + hits.append(str(p.relative_to(ROOT))) +print("checked %d secret values from %d files; files with leaks: %s" % (len(values), len(sources), hits or "none")) +sys.exit(1 if hits else 0) diff --git a/scripts/start-all.sh b/scripts/start-all.sh new file mode 100755 index 0000000..0f63dd4 --- /dev/null +++ b/scripts/start-all.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Восстанавливает остановленные API-реплики и worker (после stop-one.sh / stop-worker.sh) и ждёт готовности. +# --no-deps: хранилища и job-ы не трогаются; --no-recreate: те же контейнеры (тот же образ). +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-deps --no-recreate --wait api worker +docker compose ps --format '{{.Name}} {{.Status}}' api worker diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 0000000..f425d38 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Запустить worker после stop-worker.sh (те же контейнеры) и дождаться готовности. +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-deps --no-recreate --wait worker +docker compose ps --format '{{.Name}} {{.Status}}' worker diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..91ad5aa --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной. +# Lab7: TARGET_INSTANCE_ID = X-Instance-Id = СЛУЧАЙНЫЙ alias процесса (r-<10 hex>), не hostname. +# Реплика пишет свой alias в /tmp/instance-id (tmpfs) — сопоставляем через docker exec. +# DB, LB и volumes не трогаются. Восстановление: scripts/start-all.sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +if ! printf '%s' "$TARGET_INSTANCE_ID" | grep -Eq '^r-[0-9a-f]{10}$'; then + echo "unexpected instance id format" >&2 + exit 2 +fi +target="" +for cid in $(docker compose ps -q --status running api); do + alias=$(docker exec "$cid" cat /tmp/instance-id 2>/dev/null || true) + if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then + target=$cid + break + fi +done +[ -n "$target" ] || { echo "no running api container for $TARGET_INSTANCE_ID" >&2; exit 3; } +name=$(docker inspect -f '{{.Name}}' "$target") +echo "$(date -u +%FT%TZ) stopping api container $name ($TARGET_INSTANCE_ID)" >&2 +docker stop -t 20 "$target" >/dev/null +echo "$(date -u +%FT%TZ) stopped $name" >&2 diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 0000000..97a5323 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Сценарий защиты: остановить ВСЕ worker (SIGTERM, текущие задания дорабатываются). +# API, relay и брокер продолжают работать: upload -> 202, задания копятся в очереди. +set -eu +cd "$(dirname "$0")/.." +docker compose stop worker +docker compose ps --format '{{.Name}} {{.State}}' worker diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..031b24a 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,74 @@ -# Модель угроз — заполнить +# Модель угроз — мессенджер lab7 (Лазарев Ю. С., БПИ2303) + +Учебный стенд на одном Docker-host (Docker Desktop, macOS), синтетические данные. Модель — для заявленной оценки «3»; уровни 4/5 не заявлены, известные пробелы перечислены в «Остаточных ограничениях». ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +**Активы** + +| Актив | Где живёт | Чем защищён | +| --- | --- | --- | +| Пароли пользователей | только хеш Argon2id (m=19 MiB, t=2, p=1, соль 16 байт) в `users.password_hash` | KDF, rehash устаревших схем при входе; plaintext — только в памяти обработчика входа | +| Сессии | SHA-256 токена (256 бит энтропии) в `sessions`, TTL 3600 с | токен у клиента, в БД только хеш; отзыв logout; expiry проверяется в SQL | +| Сообщения, вложения | PostgreSQL (метаданные/тексты), SeaweedFS (байты, приватный bucket) | ACL участника чата в API (посторонний → 404), TLS ко всем хранилищам | +| Service identities | leaf-сертификаты сервисов (ключ — Docker secret только своего контейнера) | срок 30 дней, проверка SAN на стороне сервера | +| CA private keys | `$OPERATOR_DIR/ca` (вне проекта, 600) | не монтируются ни в один контейнер | +| Сервисные credentials (роли БД, ключи S3, пароли брокера) | OpenBao KV v2 `kv/messenger/` (+ исходники в `secrets/` оператора) | cert auth + policy только своего пути, TTL токена 15 мин | +| Root token и unseal shares OpenBao | `$OPERATOR_DIR/openbao/init.json` (600) | только у оператора, не в контейнерах и не в Git | +| Телеметрия | Prometheus/Loki/Tempo | mTLS на приёме, scrub атрибутов в Collector, токены/пароли/тексты не пишутся | + +**Акторы** + +- *Участник чата* — аутентифицирован Bearer, читает/пишет свой чат. +- *Посторонний пользователь* — валидная сессия, но не участник чата; цель — чужие сообщения/файлы. +- *Внешний клиент без учётной записи* — доступ только к `127.0.0.1:8443` (TLS), может перебирать пароли, слать большие тела. +- *Сетевой атакующий внутри стенда* — процесс/контейнер в одной из внутренних сетей Docker без ключей сервисов (модель «скомпрометированный соседний контейнер» или подключённый к сети `docker run --network`). +- *Скомпрометированный сервис* — атакующий получил код/ключ одного сервиса (например worker): его cert-identity, его токен OpenBao, его роль БД. +- *Оператор* — владелец хоста, CA и unseal shares; доверенный. + +**Границы учебного стенда.** Хост и Docker daemon — доверенные (root на хосте = полный доступ, в модели не защищаемся). Нет HA: один узел каждого хранилища. Нет защиты от DoS на уровне сети (только rate limit входа и лимиты ресурсов). ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +Полная матрица: [contracts/trust-matrix.md](../contracts/trust-matrix.md). + +``` + хост 127.0.0.1:8443 ──TLS(public-ca)──► lb ══mTLS(id lb)══► api×2 ──TLS verify-full──► db + │ apinet(internal) │──HTTPS──────────────► s3 + хост 127.0.0.1:3000 ──TLS(public-ca)──► grafana │──mTLS OTLP──► otel-collector ══mTLS══► prometheus/loki/tempo + │ └──TLS+cert auth──► openbao (сеть vault) + relay/worker ──AMQPS──► broker; worker ──HTTPS──► s3; все ──verify-full──► db +``` + +- **Публичные интерфейсы:** только `127.0.0.1:8443` (HTTPS API) и `127.0.0.1:3000` (HTTPS Grafana, вход admin). Остальное — в internal-сетях Docker без публикации портов. +- **Внутренние сети:** `apinet` (lb↔api), `backend` (данные), `telemetry`, `vault` (OpenBao только для процессов приложения). LB не видит БД/S3/брокер/OpenBao; Grafana не видит данные приложения. +- **Выдача секретов:** оператор → `init-secrets.sh` (PKI, пароли) → `bao-init.sh` (KV, policies, cert-роли) → процесс сервиса: свой сертификат (Docker secret) → cert login → токен со scoped policy → свой KV-путь. В env/образе секретов нет. | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | +| T1. Обход LB: контейнер в сети `apinet`/`backend` обращается к `api:8443` напрямую (без сертификата или со своим сертификатом от internal-ca, например worker/intruder) и вызывает бизнес-API, минуя rate limit и логирование LB | «Бизнес-API доступен только через LB» | mTLS на API: `CERT_REQUIRED` + SAN ∈ `MTLS_ALLOWED_CLIENTS={lb}`; порт не опубликован; health plaintext только на 127.0.0.1 | M: `LB->API[*]` — без cert, rogue CA, `intruder`, plaintext, `:8081` из сети → отказ; `lb` → 200 | Ключ LB даёт доступ к API (но не к данным без Bearer) — компрометация LB = обход rate limit | +| T2. Поддельный сервер: атакующий в сети подменяет `db`/`s3`/`openbao` (DNS-спуфинг в сети Docker) и получает пароль роли/ключ S3/токен | «Credentials уходят только настоящему серверу» | Все клиенты проверяют CA internal-ca + hostname (verify-full, `check_hostname`, `server_hostname`, `verifyhost`); fallback на plaintext нет | M: rogue CA как trust / подключение по IP / чужой ожидаемый hostname → ошибка проверки сертификата для db, s3, broker, collector, backends, openbao | Компрометация internal-ca key = возможность выпустить любой сертификат (ключ у оператора вне стенда) | +| T3. Посторонний пользователь читает чужой чат/вложение или прикрепляет чужой файл к своему сообщению, подставляет `X-User-Id` | «Доступ к чату — только участникам; identity = Bearer» | ACL в API (участие в чате проверяется в каждом запросе, 404 без раскрытия существования), `X-User-Id` игнорируется, Basic на бизнес-endpoint → 401 | A: `outsider.*` (chat/history/metadata/content/thumbnail/attach/post → 404, список чатов без чужого), `spoof.*`, `auth.basic_on_business_endpoint_401` | Участник чата может скачивать все его вложения (по контракту) | +| T4. Онлайн-перебор паролей и «дорогой login DoS» через `POST /auth/sessions` | «Вход не позволяет перебор и не выбивает реплики по CPU/RAM» | Rate limit на LB (20/10 с с IP → 429 + Retry-After, общий для реплик), Argon2id с ограничением параллелизма `PASSWORD_HASH_CONCURRENCY=2` (+503 при переполнении), одинаковый 401 и dummy-hash для неизвестного логина | A: `ratelimit.login_429_with_retry_after` (429 на 21-й попытке, `Retry-After: 10`), `auth.same_401_unknown_user_and_wrong_password`; замер стоимости хеша — evidence/argon2-bench.json | Лимит по IP: распределённый перебор с многих IP не ограничен; за NAT/Docker все клиенты хоста — один IP | +| T5. Утечка таблицы `users`/`sessions` (бэкап, SQL-инъекция в будущем коде, скомпрометированная роль) | «Из БД нельзя получить пароль или готовую сессию» | Argon2id с солью и параметрами в PHC-строке; в `sessions` — только SHA-256 токена; worker/relay/janitor вообще не имеют SELECT на `users`/`sessions` | A: `argon2.stored_format`, `argon2.unique_salt`, `argon2.rehash_legacy_scrypt_on_login`; B: worker `select_users/sessions` → denied | Офлайн-перебор слабых паролей Argon2id (19 MiB) всё ещё возможен; политика сложности паролей — только длина 12+ | +| T6. Компрометация worker (RCE через вредоносное изображение) → чтение секретов других сервисов, хешей паролей, сообщений; запись произвольных policy в OpenBao | «Worker не получает admin/чужих credentials» | Отдельная cert-identity и policy `messenger-worker` (read только `kv/data/messenger/worker`); роль БД `worker_svc` без доступа к users/sessions/chats/messages; свой S3-ключ; transform в подпроцессе с RLIMIT, контейнер non-root, read-only, без capabilities, лимиты CPU/RAM/PIDs | B: worker — чужой путь, `migrate`, list, запись, policy, token create, seal → 403; S3-ключ = worker, не admin/api; DB denied; H: `worker.*` (uid 10001, cap_drop ALL, read-only) | Worker читает все вложения bucket (права по bucket, не по prefix) и может писать в jobs/attachments | +| T7. Утечка секретов через репозиторий, образ, env или телеметрию (Bearer в логах, пароль в атрибутах span, ключ в слое образа) | «Секреты не покидают своих хранилищ» | `.gitignore`/`.dockerignore` (secrets/, *.key, *.pem), секреты только из OpenBao (в env — нет), scrub в Collector, приложение не логирует заголовки/тела | H: `repo.scan_secrets_clean` (125 значений, вкл. CA keys/root token/shares), `image.*`, `*.no_secret_values_in_env`, `telemetry.loki_no_*` с положительным контролем, `telemetry.tempo_no_token_or_password` | Root на хосте видит всё (`docker inspect`, `/run/secrets`) | +| T8. Попытка получить root/широкий токен OpenBao из приложения или переиспользовать украденный токен долго | «Приложение не держит root/безграничный токен» | cert auth → service token, TTL 15 мин, max 1 ч; нет BAO_TOKEN/SecretID в env; revoke-self работает сразу | B: `*.no_token_in_env`, `token_ttl_bounded`, `deny_create_token`, `revoked_token_denied` | Украденный ключ сервиса = повторный login до истечения сертификата (30 дней) или удаления роли; CRL/OCSP не реализованы | +| T9. Подмена/чтение телеметрии: произвольный контейнер шлёт фальшивые метрики/логи в Collector или читает Loki/Prometheus | «Ingest и чтение телеметрии — только у сервисов стенда» | mTLS на Collector и всех backend-ах (client cert internal-ca обязателен), сети `telemetry` internal | M: Collector/Prometheus/Loki/Tempo/lb:8404 без cert, rogue CA → отказ | Любой сертификат internal-ca принимается (role не проверяется) — уровень 4 | +| T10. Большие тела запросов / zip-бомба изображения истощают память API | «Реплика не падает по OOM от одного клиента» | 1 MiB JSON, 10 MiB upload (Content-Length до чтения), 40 Мпикс до декодирования, подпроцесс проверки JPEG с RLIMIT_AS, cgroup-лимиты, tmpfs 160 MiB | A: `limits.json_body_over_1mib_413`, `limits.upload_over_10mib_413`; lab5: transform_limits | Медленные клиенты (slowloris) ограничены таймаутами LB, отдельно в lab7 не тестировались | На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +**Bootstrap.** Оператор: `./scripts/init-secrets.sh` — пароли сервисов и PKI (OpenSSL закреплённого образа, `--network none`; ключи CA → `$OPERATOR_DIR/ca`, leaf → `secrets/pki`). `./scripts/bao-init.sh` — `operator init` (3 shares, порог 2 → `$OPERATOR_DIR/openbao/init.json`), unseal, KV v2, policies, роли auth/cert, запись KV. Первый секрет для доступа к OpenBao у сервиса — его собственный leaf-ключ (Docker secret); отдельного SecretID нет. Trust anchor — `internal-ca.crt`. + +**Expiry/renewal.** Токен OpenBao: renew-self на половине TTL, после max TTL — новый login (src/vault.py). Сертификаты leaf — 30 дней, CA — 365 дней; продление: удалить leaf-файлы и снова `init-secrets.sh` (выпустит недостающие), затем `docker compose up -d --force-recreate `. Автоматического renewal сертификатов нет (уровень 4/5). + +**Revoke/rotation.** Отзыв identity сервиса в OpenBao — `bao delete auth/cert/certs/` (новые логины невозможны), отзыв действующих токенов — `bao token revoke -mode=path auth/cert/login`. Для mTLS к API отзыв = исключение SAN из `MTLS_ALLOWED_CLIENTS` или смена CA; CRL не используется. Ротация секрета: новое значение в `secrets/` → `bao-init.sh` (перезапишет KV) → для роли БД `docker compose up migrate` → рестарт сервиса (новый login читает новое значение). В рамках «3» не демонстрировалось. + +**Компрометация identity.** Пример worker: удалить роль `auth/cert/certs/worker`, отозвать токены, выпустить новый leaf worker, сменить пароли `worker_svc`/S3-ключ worker/брокера в `secrets/`, `bao-init.sh`, `migrate`, пересоздать broker/s3 (identities/definitions), пересоздать worker. Задания не теряются: они в outbox/очереди и повторно доставляются (at-least-once, lab5). + +**Sealed OpenBao и восстановление.** После рестарта контейнера OpenBao — sealed (B: `sealed.after_restart_sealed`). Работающие процессы продолжают обслуживать (секреты в памяти; renew падает → предупреждение `vault.renew_failed`, повтор с backoff до 30 с). Новые/перезапущенные процессы не становятся ready (B: `sealed.restarted_replica_not_healthy`), через 300 с без OpenBao завершаются и перезапускаются Docker. Восстановление: оператор `./scripts/bao-unseal.sh` → реплика healthy за ~12 с (B). Данные KV — на Raft-volume (B: `sealed.kv_persisted_after_restart`). Владелец CA и unseal — оператор (единственный человек с доступом к `$OPERATOR_DIR`); unseal shares хранятся отдельно от OpenBao, но на том же хосте — это учебный компромисс (в production — разные хранители shares/KMS auto-unseal). + +**Scan findings.** Dependency/image scan (pip-audit/Trivy) с разбором применимости — пункт уровня 4, в этой работе не выполнялся. Учтённые известные проблемы зависимостей унаследованы из lab5 (Pillow 13.0.0 — GHSA-wqgw-9xfh-qf2g, urllib3 2.8.0 — GHSA-vxq7-64xx-v4gw). Новая зависимость lab7 — `argon2-cffi 25.1.0` (последняя версия на момент работы). diff --git a/src/common.py b/src/common.py new file mode 100644 index 0000000..5fe984e --- /dev/null +++ b/src/common.py @@ -0,0 +1,107 @@ +"""Общее для фоновых процессов lab5 (relay, worker, ops): конфигурация, БД, брокер. + +API (server.py) брокер не использует вовсе: он только пишет outbox в PostgreSQL. +Поэтому недоступность RabbitMQ не влияет на приём файлов — задания копятся в outbox. +""" +import logging +import os +import socket +import sys +import time +import uuid + +import pika +from psycopg.conninfo import make_conninfo + +import tlsutil +import vault +from instance import INSTANCE_ID + +# Lab6: логирование (JSON stdout + OTLP) настраивает telemetry.setup() в main() каждого процесса. +logging.getLogger("pika").setLevel(logging.WARNING) + +# Топология брокера (объявляется декларативно в secrets/rabbitmq_definitions.json, +# приложения её не создают — у их пользователей нет права configure). +EXCHANGE = "attachments" +ROUTING_KEY = "attachment.process.v1" +QUEUE = "attachments.process.v1" +DLQ = "attachments.dlq" + +HOSTNAME = socket.gethostname() +# Идентификатор процесса для lease_owner: hostname контейнера + случайный boot id. +BOOT_ID = uuid.uuid4().hex[:8] +OWNER = "%s:%s" % (HOSTNAME, BOOT_ID) + + +def env_int(name, default): + return int(os.environ.get(name, str(default))) + + +def env_float(name, default): + return float(os.environ.get(name, str(default))) + + +def secret(name): + """Lab7: из OpenBao (src/vault.py); NAME_FILE/NAME — только для инфраструктурных job/тестов.""" + return vault.secret(name) + + +def database_conninfo(): + options = "-c statement_timeout=%d" % env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + url = vault.secret("DATABASE_URL", required=False) + if url: + return make_conninfo(url, connect_timeout=3, options=options, sslmode="verify-full", + sslrootcert=os.environ.get("DB_SSLROOTCERT", tlsutil.CA_FILE)) + return make_conninfo( + host=os.environ.get("POSTGRES_HOST", "localhost"), + port=os.environ.get("POSTGRES_PORT", "5432"), + dbname=os.environ.get("POSTGRES_DB", "messenger"), + user=os.environ.get("POSTGRES_USER", "messenger"), + password=secret("POSTGRES_PASSWORD"), + connect_timeout=3, + sslmode=os.environ.get("DB_SSLMODE", "verify-full"), + sslrootcert=os.environ.get("DB_SSLROOTCERT", tlsutil.CA_FILE), + options=options, + ) + + +def broker_parameters(): + """amqp URL из BROKER_URL_FILE: у relay, worker и ops — разные пользователи RabbitMQ.""" + params = pika.URLParameters(secret("BROKER_URL")) + # Lab7: AMQPS (5671) — проверка CA и hostname брокера; клиентский сертификат сервиса + # предъявляется (RabbitMQ verify_peer). Plaintext 5672 у брокера выключен. + if params.port == 5671 or os.environ.get("BROKER_TLS", "1") == "1": + params.ssl_options = pika.SSLOptions(tlsutil.client_context(), + server_hostname=os.environ.get("BROKER_TLS_SERVER_NAME", params.host)) + # heartbeat: брокер закроет соединение мёртвого/замороженного клиента и вернёт его unacked + # сообщения в очередь (в т.ч. выданные заранее по prefetch). 10 с: эксперимент + # lease_takeover показал, что при 20 с такое сообщение висело ~35 с. Worker обрабатывает + # задание в отдельном потоке и продолжает обслуживать heartbeat (process_data_events), + # поэтому долгая обработка соединение не рвёт. + params.heartbeat = env_int("BROKER_HEARTBEAT_SECONDS", 10) + params.blocked_connection_timeout = 30 + params.socket_timeout = 5 + params.connection_attempts = 1 + params.client_properties = {"connection_name": "%s-%s" % (os.environ.get("SERVICE_NAME", "app"), HOSTNAME)} + return params + + +def connect_broker(log, stop=None): + """Подключение с повторами и паузой до 5 с; stop — threading.Event для выхода.""" + delay = 0.5 + while True: + try: + return pika.BlockingConnection(broker_parameters()) + except Exception as exc: + log.warning("broker unavailable (%s), retry in %.1fs", type(exc).__name__, delay, + extra={"event": "broker.unavailable", "error.type": type(exc).__name__}) + if stop is not None and stop.wait(delay): + return None + if stop is None: + time.sleep(delay) + delay = min(5.0, delay * 2) + + +def touch(path): + with open(path, "w") as f: + f.write(str(time.time())) diff --git a/src/images.py b/src/images.py new file mode 100644 index 0000000..c7f6eb3 --- /dev/null +++ b/src/images.py @@ -0,0 +1,282 @@ +"""Проверка загружаемых изображений по СОДЕРЖИМОМУ, с ограниченной памятью. + +Заголовок Content-Type клиента — только заявление: настоящий формат определяется +по сигнатуре и структуре файла. Лимит пикселей проверяется по заголовку (PNG IHDR, +JPEG SOF) ДО распаковки, поэтому «бомба» 100 000×100 000 не декодируется вовсе. + +PNG проверяется собственным потоковым валидатором (стандартная библиотека): +структура и CRC каждого chunk, допустимые IHDR, IDAT распаковывается zlib кусками +по 1 MiB без сохранения результата; объём распакованных данных должен ТОЧНО совпасть +с размером, который следует из IHDR (больше — бомба/мусор, меньше — обрезанный файл). + +JPEG: разбор маркеров до SOF (размеры), затем пробное декодирование Pillow в режиме +draft (масштаб 1/8) в ОТДЕЛЬНОМ процессе с лимитами (src/jpeg_probe.py). Для baseline JPEG +draft экономит память, но progressive JPEG держит DCT-коэффициенты всего кадра: до ~250 MiB +на 40 Мпикс (аудит F02) — поэтому не в процессе API. + +PNG (аудит F05): кроме CRC/zlib/объёма проверяется байт фильтра КАЖДОЙ строки (0..4), +включая проходы Adam7, а также PLTE (кратна 3, <= 256 записей) и пустой IEND. +""" +import os +import subprocess +import sys +import struct +import zlib + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SOI = b"\xff\xd8\xff" +READ_CHUNK = 64 * 1024 +INFLATE_STEP = 1024 * 1024 + +# Допустимые глубины цвета по типу цвета PNG и число каналов. +_PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +# Adam7: (x0, y0, dx, dy) +_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +_JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + + +class InvalidImage(Exception): + """Файл не является корректным PNG/JPEG заявленного типа -> 415.""" + + +class TooManyPixels(Exception): + """Декодированный размер превышает лимит -> 413.""" + + +class ResourceLimit(Exception): + """Проверка не уложилась в лимиты памяти/времени -> 413 (вход слишком тяжёлый).""" + + +JPEG_PROBE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "jpeg_probe.py") +JPEG_PROBE_TIMEOUT_SECONDS = float(os.environ.get("JPEG_PROBE_TIMEOUT_SECONDS", "15")) + + +def sniff(head): + """Тип по сигнатуре первых байтов: 'image/png', 'image/jpeg' или None.""" + if head.startswith(PNG_SIGNATURE): + return "image/png" + if head.startswith(JPEG_SOI): + return "image/jpeg" + return None + + +def png_header_pixels(head): + """Ранняя проверка по первым 33 байтам (сигнатура + IHDR): (w, h) или None.""" + if len(head) < 33 or not head.startswith(PNG_SIGNATURE) or head[12:16] != b"IHDR": + return None + return struct.unpack(">II", head[16:24]) + + +def _read_exact(f, n): + data = f.read(n) + if len(data) != n: + raise InvalidImage("truncated") + return data + + +def _png_passes(w, h, bits_per_pixel, interlace): + """[(строк, байт в строке вместе с байтом фильтра)] для каждого непустого прохода.""" + def row(pw): + return 1 + (pw * bits_per_pixel + 7) // 8 + if interlace == 0: + return [(h, row(w))] + out = [] + for x0, y0, dx, dy in _ADAM7: + pw = (w - x0 + dx - 1) // dx if w > x0 else 0 + ph = (h - y0 + dy - 1) // dy if h > y0 else 0 + if pw and ph: + out.append((ph, row(pw))) + return out + + +class _FilterCheck: + """Байт фильтра в начале каждой строки распакованных данных PNG должен быть 0..4. + Проверка срезами bytes[first::rowbytes] — скорость C, без цикла Python по строкам + (иначе PNG 1x40 000 000 стал бы атакой на CPU).""" + + def __init__(self, passes): + self.passes = passes + self.index = 0 + self.offset = 0 # байт уже пройдено в текущем проходе + + def feed(self, out): + i, n = 0, len(out) + while i < n: + if self.index >= len(self.passes): + raise InvalidImage("too much pixel data") + rows, rb = self.passes[self.index] + total = rows * rb + take = min(n - i, total - self.offset) + first = (-self.offset) % rb + if first < take and max(out[i + first:i + take:rb]) > 4: + raise InvalidImage("bad filter type") + i += take + self.offset += take + if self.offset == total: + self.index += 1 + self.offset = 0 + + +def _png_expected_bytes(w, h, bits_per_pixel, interlace): + def raw(pw, ph): + return ph * (1 + (pw * bits_per_pixel + 7) // 8) if pw and ph else 0 + if interlace == 0: + return raw(w, h) + total = 0 + for x0, y0, dx, dy in _ADAM7: + pw = (w - x0 + dx - 1) // dx if w > x0 else 0 + ph = (h - y0 + dy - 1) // dy if h > y0 else 0 + total += raw(pw, ph) + return total + + +def validate_png(f, size, max_pixels): + if _read_exact(f, 8) != PNG_SIGNATURE: + raise InvalidImage("signature") + pos = 8 + width = height = None + expected = produced = 0 + inflater = None + seen_plte = idat_done = False + color_type = None + last_type = None + while True: + if pos + 12 > size: + raise InvalidImage("truncated") + length, ctype = struct.unpack(">I4s", _read_exact(f, 8)) + pos += 8 + if length > 0x7FFFFFFF or pos + length + 4 > size: + raise InvalidImage("chunk length") + if not ctype.isalpha(): + raise InvalidImage("chunk type") + if width is None and ctype != b"IHDR": + raise InvalidImage("IHDR must be first") + crc = zlib.crc32(ctype) + if ctype == b"IDAT": + if idat_done or (color_type == 3 and not seen_plte): + raise InvalidImage("IDAT order") + remaining = length + while remaining: + piece = _read_exact(f, min(READ_CHUNK, remaining)) + remaining -= len(piece) + crc = zlib.crc32(piece, crc) + buf = piece + while buf: + if inflater.eof: + raise InvalidImage("data after zlib stream") + try: + out = inflater.decompress(buf, INFLATE_STEP) + except zlib.error: + raise InvalidImage("zlib") + produced += len(out) + if produced > expected: + raise InvalidImage("too much pixel data") # бомба/мусор + filters.feed(out) + buf = inflater.unconsumed_tail + else: + if last_type == b"IDAT": + idat_done = True + data = _read_exact(f, length) + crc = zlib.crc32(data, crc) + if ctype == b"IHDR": + if width is not None or length != 13: + raise InvalidImage("IHDR") + width, height, depth, color_type, comp, filt, interlace = struct.unpack(">IIBBBBB", data) + if not (1 <= width <= 0x7FFFFFFF and 1 <= height <= 0x7FFFFFFF): + raise InvalidImage("dimensions") + if width * height > max_pixels: + raise TooManyPixels() + if depth not in _PNG_DEPTHS.get(color_type, ()) or comp != 0 or filt != 0 or interlace not in (0, 1): + raise InvalidImage("IHDR values") + expected = _png_expected_bytes(width, height, _PNG_CHANNELS[color_type] * depth, interlace) + filters = _FilterCheck(_png_passes(width, height, _PNG_CHANNELS[color_type] * depth, interlace)) + inflater = zlib.decompressobj() + elif ctype == b"PLTE": + # PLTE: кратна 3, 1..256 записей, только одна, не для серых типов, до IDAT. + if seen_plte or length == 0 or length % 3 or length > 768 or color_type in (0, 4) \ + or last_type == b"IDAT" or idat_done: + raise InvalidImage("PLTE") + seen_plte = True + elif ctype == b"IEND": + if length != 0: + raise InvalidImage("IEND") + elif ctype[0:1].isupper(): + raise InvalidImage("unknown critical chunk") # декодер обязан его понимать + if struct.unpack(">I", _read_exact(f, 4))[0] != crc & 0xFFFFFFFF: + raise InvalidImage("crc") + pos += length + 4 + last_type = ctype + if ctype == b"IEND": + break + if inflater is None or not inflater.eof or produced != expected: + raise InvalidImage("incomplete pixel data") + if pos != size: + raise InvalidImage("trailing data") # полиглоты PNG+что-то ещё не принимаем + return width, height + + +def jpeg_dimensions(f, size): + """Размеры из первого SOF без декодирования. Ограничено размером файла.""" + if _read_exact(f, 2) != b"\xff\xd8": + raise InvalidImage("signature") + pos = 2 + while pos < size: + b = _read_exact(f, 1) + pos += 1 + if b != b"\xff": + raise InvalidImage("marker") + marker = 0xFF + while marker == 0xFF: + marker = _read_exact(f, 1)[0] + pos += 1 + if marker == 0x01 or 0xD0 <= marker <= 0xD7: + continue + if marker in (0xD8, 0xD9, 0xDA): + raise InvalidImage("no SOF before scan") + seg_len = struct.unpack(">H", _read_exact(f, 2))[0] + if seg_len < 2 or pos + seg_len > size: + raise InvalidImage("segment length") + if marker in _JPEG_SOF: + if seg_len < 8: + raise InvalidImage("SOF") + _precision, height, width = struct.unpack(">BHH", _read_exact(f, 5)) + if width == 0 or height == 0: + raise InvalidImage("dimensions") # DNL-высота не поддерживается + return width, height + f.seek(seg_len - 2, 1) + pos += seg_len + raise InvalidImage("no SOF") + + +def validate_jpeg(f, size, max_pixels): + width, height = jpeg_dimensions(f, size) + if width * height > max_pixels: + raise TooManyPixels() + if not hasattr(f, "fileno"): + raise InvalidImage("probe needs a real file") + f.flush() + f.seek(0) + try: + # stdin подпроцесса — тот же временный файл (анонимный, O_TMPFILE): байты не копируются. + proc = subprocess.run([sys.executable, JPEG_PROBE, str(max_pixels), str(width), str(height)], + stdin=f, capture_output=True, timeout=JPEG_PROBE_TIMEOUT_SECONDS, check=False) + except subprocess.TimeoutExpired: + raise ResourceLimit("timeout") + finally: + f.seek(0) + if proc.returncode == 0: + return width, height + if proc.returncode == 3: + raise InvalidImage("decode") + raise ResourceLimit("probe exit %d" % proc.returncode) # 5 или убит сигналом (RLIMIT/OOM) + + +def validate(f, size, content_type, max_pixels): + """(width, height) или InvalidImage/TooManyPixels. f — seekable файл, позиция 0.""" + f.seek(0) + if content_type == "image/png": + return validate_png(f, size, max_pixels) + if content_type == "image/jpeg": + return validate_jpeg(f, size, max_pixels) + raise InvalidImage("type") diff --git a/src/instance.py b/src/instance.py new file mode 100644 index 0000000..f6c9475 --- /dev/null +++ b/src/instance.py @@ -0,0 +1,20 @@ +"""Lab7: alias процесса для X-Instance-Id и service.instance.id. + +Случайное значение на жизнь процесса: не раскрывает hostname/IP/ID контейнера. Чтобы +scripts/stop-one.sh мог найти контейнер по alias, процесс пишет его в /tmp/instance-id +(tmpfs контейнера; читается только через docker exec оператором). +""" +import os +import secrets + +if os.environ.get("INSTANCE_ID"): + # Унаследован от родителя (подпроцессы проверки JPEG/transform) — тот же alias, файл не трогаем. + INSTANCE_ID = os.environ["INSTANCE_ID"] +else: + INSTANCE_ID = "r-" + secrets.token_hex(5) + os.environ["INSTANCE_ID"] = INSTANCE_ID + try: + with open(os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id"), "w") as _f: + _f.write(INSTANCE_ID) + except OSError: + pass diff --git a/src/janitor.py b/src/janitor.py new file mode 100644 index 0000000..1b64c25 --- /dev/null +++ b/src/janitor.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Сверка PostgreSQL <-> S3 и очистка (compose-сервис `janitor`). Lab5. + +Prefix-ы bucket: + staging/ принятые оригиналы до публикации результата (attachments.staging_key); + attachments//v/ конечные объекты: original и варианты; + originals/ оригиналы lab4 (после миграции 004 — staging-объекты backfill-заданий); + health/, bootstrap/ служебные, не трогаются. + +За один проход (под pg_try_advisory_lock): + 1. abandoned: pending-строки старше grace (API упал до постановки в очередь, 202 не было) + -> удалить staging-объект, затем строку; + 2. staging ready: у ready-вложения остался staging (worker не смог удалить после COMMIT) + -> удалить объект, staging_key = NULL; + 3. staging failed: failed дольше STAGING_RETENTION_SECONDS -> удалить staging (replay после + этого невозможен: ops replay ответит refused_staging_expired). Для 2 и 3: под блокировкой + jobs+attachments условие проверяется заново, staging_key снимается и COMMIT, потом DELETE + в S3 — гонка с replay исключена (аудит F01); + 4. orphan: объекты staging/, originals/, attachments/ старше grace, на которые не ссылается + ни одна строка (staging_key, object_key ready-вложения, attachment_variants), и вложение + не queued/processing -> удалить; + 5. missing: ready без объекта оригинала или варианта -> ТОЛЬКО отчёт. +Текущую работу (queued/processing) janitor не удаляет никогда: её staging нужен worker. +Grace (ATTACHMENT_GRACE_SECONDS, 3600 с) многократно больше UPLOAD_TIMEOUT_SECONDS и +JOB_TIMEOUT_SECONDS, поэтому идущая загрузка/обработка не считается брошенной. + + python src/janitor.py --once --dry-run | --once | --interval 300 +""" +import argparse +import datetime as dt +import json +import logging +import os +import re +import sys +import time + +import jobs +import telemetry +from server import OBJ, STORE + +log = logging.getLogger("messenger.janitor") +JANITOR_LOCK_ID = 727002 +PREFIXES = ("staging/", "originals/", "attachments/") +HEARTBEAT = "/tmp/janitor.heartbeat" +STAGING_RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")) +# Только для теста гонки (student-tests/janitor_replay_race.py): пауза между выборкой +# кандидатов и их захватом — воспроизводит чередование «выбрал -> replay -> удаляю». Обычно 0. +PAUSE_AFTER_SELECT = float(os.environ.get("JANITOR_PAUSE_AFTER_SELECT_SECONDS", "0")) +_FINAL_RE = re.compile(r"^attachments/([0-9a-f-]{36})/") + + +def heartbeat(): + with open(HEARTBEAT, "w") as f: + f.write(str(time.time())) + + +def _delete(report, key, dry_run): + if dry_run: + return True + try: + OBJ.delete(key) + report["deleted_objects"] += 1 + return True + except Exception as exc: + report["errors"] += 1 + log.warning("object not deleted (%s)", type(exc).__name__) + return False + + +def run_once(grace_seconds, dry_run, staging_retention=STAGING_RETENTION): + started = time.monotonic() + report = {"dry_run": dry_run, "grace_seconds": grace_seconds, "staging_retention_seconds": staging_retention, + "abandoned_pending": 0, "staging_ready_cleaned": 0, "staging_failed_expired": 0, + "staging_skipped_changed": 0, + "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_rows": 0, "errors": 0} + with STORE.pool.connection() as conn: + conn.autocommit = True + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK_ID,)).fetchone()["ok"]: + report["skipped"] = "another janitor is running" + return report + try: + now_db = conn.execute("SELECT now() AS t").fetchone()["t"] + cutoff = now_db - dt.timedelta(seconds=grace_seconds) + report["rows_by_status"] = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, count(*) AS n FROM attachments GROUP BY status")} + report["jobs_by_status"] = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, count(*) AS n FROM jobs GROUP BY status")} + report["outbox_unpublished"] = conn.execute( + "SELECT count(*) AS n FROM outbox WHERE published_at IS NULL").fetchone()["n"] + + # 1. брошенные pending + abandoned = conn.execute( + """SELECT id, staging_key FROM attachments + WHERE status = 'pending' AND created_at < %s ORDER BY created_at""", (cutoff,)).fetchall() + report["abandoned_pending"] = len(abandoned) + report["abandoned_ids"] = [str(r["id"]) for r in abandoned[:20]] + for row in abandoned: + if dry_run: + continue + if row["staging_key"] and not _delete(report, row["staging_key"], dry_run): + continue + report["deleted_rows"] += conn.execute( + "DELETE FROM attachments WHERE id = %s AND status = 'pending'", (row["id"],)).rowcount + + # 2/3. staging, который больше не нужен + stale = conn.execute( + """SELECT a.id, a.staging_key, a.status, j.id AS job_id FROM attachments a + LEFT JOIN jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.staging_key IS NOT NULL AND (a.status = 'ready' + OR (a.status = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))""", + (staging_retention,)).fetchall() + report["staging_candidates"] = [str(r["id"]) for r in stale][:20] + if stale and PAUSE_AFTER_SELECT > 0: + time.sleep(PAUSE_AFTER_SELECT) + for row in stale: + if dry_run: + report["staging_ready_cleaned" if row["status"] == "ready" else "staging_failed_expired"] += 1 + continue + # Аудит F01: выборка выше — без блокировок; между ней и удалением оператор мог + # выполнить replay (failed -> queued), и удаление уничтожило бы исходник принятой + # работы. Поэтому под блокировкой строк (порядок как у replay и worker: + # сначала jobs, потом attachments) условие проверяется ЗАНОВО, ссылка + # staging_key снимается и фиксируется, и только ПОСЛЕ COMMIT удаляется объект. + # Replay после этого увидит staging_key IS NULL и откажет (refused_staging_expired); + # replay, успевший раньше, оставит queued — и janitor пропустит строку. + with conn.transaction(): + conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s FOR UPDATE", (row["id"],)) + claimed = conn.execute( + """UPDATE attachments SET staging_key = NULL + WHERE id = %s AND staging_key = %s AND (status = 'ready' + OR (status = 'failed' AND updated_at < now() - make_interval(secs => %s))) + RETURNING status""", + (row["id"], row["staging_key"], staging_retention)).fetchone() + if claimed is None: + report["staging_skipped_changed"] += 1 # состояние изменилось (replay и т. п.) + continue + if row["job_id"]: + jobs.add_event(conn, row["job_id"], "staging_released", None, "janitor", + {"status": claimed["status"]}) + report["staging_ready_cleaned" if claimed["status"] == "ready" else "staging_failed_expired"] += 1 + # Ссылки уже нет: если удаление не удастся, объект станет сиротой и будет удалён + # обходом bucket после grace — исходник «живой» работы так не потеряется. + _delete(report, row["staging_key"], dry_run) + + # 4/5. обход bucket. Момент начала — по часам БД: ready раньше этого момента обязан + # иметь все объекты в листинге (объекты пишутся ДО COMMIT ready). + list_started = conn.execute("SELECT now() AS t").fetchone()["t"] + listed = {} + for prefix in PREFIXES: + for obj in OBJ.client.list_objects(OBJ.bucket, prefix=prefix, recursive=True): + listed[obj.object_name] = obj.last_modified + report["objects_total"] = len(listed) + report["objects_by_prefix"] = {p: sum(1 for k in listed if k.startswith(p)) for p in PREFIXES} + known = set() + busy = set() # вложения с текущей работой: их объекты не трогаем + keys = list(listed) + for i in range(0, len(keys), 1000): + batch = keys[i:i + 1000] + for r in conn.execute( + """SELECT staging_key AS k FROM attachments WHERE staging_key = ANY(%(b)s) + UNION SELECT object_key FROM attachments WHERE status = 'ready' AND object_key = ANY(%(b)s) + UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%(b)s)""", + {"b": batch}): + known.add(r["k"]) + ids = {m.group(1) for k in keys for m in [_FINAL_RE.match(k)] if m} + if ids: + for r in conn.execute( + "SELECT id FROM attachments WHERE id = ANY(%s::uuid[]) AND status IN ('pending','queued','processing')", + (list(ids),)): + busy.add(str(r["id"])) + orphans = [] + for k, mtime in listed.items(): + if k in known or mtime is None or mtime >= cutoff: + continue + m = _FINAL_RE.match(k) + if m and m.group(1) in busy: + continue + orphans.append(k) + report["orphan_objects"] = len(orphans) + report["orphan_keys_sample"] = sorted(orphans)[:5] + for key in orphans: + _delete(report, key, dry_run) + + missing = [] + for r in conn.execute( + """SELECT a.id, a.object_key AS k FROM attachments a WHERE a.status = 'ready' AND a.updated_at < %s + UNION ALL + SELECT v.attachment_id, v.object_key FROM attachment_variants v + JOIN attachments a ON a.id = v.attachment_id + WHERE a.status = 'ready' AND a.updated_at < %s""", (list_started, list_started)): + if r["k"] not in listed: + missing.append(str(r["id"])) + report["missing_objects"] = len(missing) + report["missing_ids_sample"] = sorted(set(missing))[:5] + if missing: + log.error("ready attachments without object: %d (manual investigation needed)", len(missing)) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK_ID,)) + report["duration_ms"] = round((time.monotonic() - started) * 1000, 1) + return report + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--once", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--grace-seconds", type=int, + default=int(os.environ.get("ATTACHMENT_GRACE_SECONDS", "3600"))) + parser.add_argument("--staging-retention-seconds", type=int, default=STAGING_RETENTION) + parser.add_argument("--interval", type=int, default=int(os.environ.get("JANITOR_INTERVAL_SECONDS", "300"))) + args = parser.parse_args() + if args.grace_seconds < 1: + parser.error("grace period must be positive") + if args.once: + telemetry.configure_logging(stream=sys.stderr) # stdout — JSON-отчёт для оператора + else: + telemetry.setup("messenger-janitor") + telemetry.observe_pool(STORE.pool, "janitor") + STORE.open() + try: + while True: + try: + print(json.dumps(run_once(args.grace_seconds, args.dry_run, args.staging_retention_seconds), + ensure_ascii=False), flush=True) + except Exception as exc: + log.warning("janitor pass failed (%s)", type(exc).__name__, + extra={"event": "janitor.failed", "error.type": type(exc).__name__}) + if args.once: + return 1 + if args.once: + return 0 + wake = time.monotonic() + args.interval + while time.monotonic() < wake: + heartbeat() + time.sleep(min(30, max(0.1, wake - time.monotonic()))) + finally: + STORE.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..30c9a4d --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,451 @@ +"""Задания обработки вложений: схема сообщения, переходы состояний, retry, lease, replay. + +Используется API (постановка в очередь), worker, relay (reaper) и ops-CLI. Все переходы — +SQL-транзакции над строкой jobs под FOR UPDATE, поэтому решение «что делать с сообщением» +принимается по устойчивому состоянию в PostgreSQL, а не по факту доставки. + +Сообщение брокера (contracts/jobs.schema.json) — только ссылка: job_id, attachment_id, +pipeline_version, attempt. Bytes, credentials, URL и ключи S3 в сообщение не попадают: +worker берёт доверенные storage metadata из БД. + +Переходы (contracts/attachment-states.md): + enqueue нет -> queued API, одна транзакция с outbox + claim queued -> processing worker: attempt+1, lease_owner/token/until + heartbeat processing (продление) worker, только со своим lease_token + complete processing -> ready worker, только со своим lease_token (fencing) + retry processing -> queued временная ошибка, backoff + jitter, outbox с available_at + fail processing -> failed постоянная ошибка или исчерпаны попытки + lease expired processing -> queued reaper (relay), если аренду никто не продлил + replay failed -> queued оператор (ops.py), audit, ready не трогается + +Lab6 (W3C Trace Context через очередь): add_outbox внутри активной трассы создаёт PRODUCER span +«attachment.process.v1 create» и пишет его traceparent/tracestate в payload outbox (поля +разрешены contracts/jobs.schema.json). Relay публикует сообщение в span-е «send» (child), worker +обрабатывает в CONSUMER span-е «process» (child от send). Retry из worker пишет новый outbox в +контексте текущей попытки -> следующая попытка в той же трассе. Фоновые переходы без трассы +(reaper, redrive, replay оператора) берут traceparent последнего outbox этого job — цепочка +upload -> все попытки остаётся одной трассой. +""" +import datetime as dt +import json +import random +import re +import uuid + +import telemetry +from telemetry import SpanKind, db_operation + +SCHEMA_VERSION = 1 +# Аудит F11: публикация результата и выборка вариантов согласованы только для v1. +SUPPORTED_PIPELINE_VERSIONS = (1,) +JOB_TYPE = "attachment.process.v1" +VARIANT_NAMES = ("thumbnail", "crop", "optimized") +# Постоянные ошибки: повтор не поможет (тот же вход даст тот же результат). +PERMANENT_ERRORS = {"invalid_image", "image_too_large", "staging_corrupted", "staging_missing", + "resource_limit"} +_UUID_FIELDS = ("job_id", "attachment_id") +_ALLOWED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", + "created_at", "traceparent", "tracestate"} + + +class InvalidMessage(Exception): + """Сообщение не соответствует jobs.schema.json -> reject в DLQ.""" + + +def utc_now_text(): + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def make_payload(job_id, attachment_id, pipeline_version, attempt, created_at=None, trace_fields=None): + payload = { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created_at or utc_now_text(), + } + for key in ("traceparent", "tracestate"): + value = (trace_fields or {}).get(key) + if key == "traceparent" and value and _TRACEPARENT_RE.match(value): + payload[key] = value + elif key == "tracestate" and value and len(value) <= 512: + payload[key] = value + return payload + + +_UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") +# RFC 3339 date-time: обязательно время и зона (Z или ±hh:mm). +_DATETIME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$") +_TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + + +def _strict_int(v): + # В Python True == 1 и isinstance(True, int): bool исключается явно (аудит F12). + # По JSON Schema 2020-12 integer — число с нулевой дробной частью: 1.0 тоже допустимо. + return type(v) is int or (type(v) is float and v.is_integer()) + + +def parse_message(body): + """bytes -> dict строго по jobs.schema.json (без внешних библиотек). Иначе InvalidMessage. + Аудит F12: раньше проходили schema_version=true, created_at без времени, traceparent-число + и tracestate длиннее 512. Соответствие сверяется с jsonschema тестом job_schema_check.py.""" + try: + msg = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidMessage("not json") + if not isinstance(msg, dict): + raise InvalidMessage("not object") + if set(msg) - _ALLOWED: + raise InvalidMessage("extra fields") + for field in ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"): + if field not in msg: + raise InvalidMessage("missing " + field) + if not _strict_int(msg["schema_version"]) or msg["schema_version"] != SCHEMA_VERSION: + raise InvalidMessage("unsupported schema_version") + if msg["type"] != JOB_TYPE: + raise InvalidMessage("unsupported type") + for field in _UUID_FIELDS: + if not isinstance(msg[field], str) or not _UUID_RE.match(msg[field]): + raise InvalidMessage("bad " + field) + for field in ("pipeline_version", "attempt"): + if not _strict_int(msg[field]) or msg[field] < 1: + raise InvalidMessage("bad " + field) + created = msg["created_at"] + if not isinstance(created, str) or not _DATETIME_RE.match(created): + raise InvalidMessage("bad created_at") + try: + dt.datetime.fromisoformat(created.upper().replace("Z", "+00:00")) + except ValueError: + raise InvalidMessage("bad created_at") + if "traceparent" in msg and (not isinstance(msg["traceparent"], str) or not _TRACEPARENT_RE.match(msg["traceparent"])): + raise InvalidMessage("bad traceparent") + if "tracestate" in msg and (not isinstance(msg["tracestate"], str) or len(msg["tracestate"]) > 512): + raise InvalidMessage("bad tracestate") + for field in ("schema_version", "pipeline_version", "attempt"): + msg[field] = int(msg[field]) # 1.0 -> 1 + return msg + + +def backoff_seconds(attempt, base, cap, rng=random): + """Экспоненциальный backoff с «equal jitter»: d = min(cap, base*2^(attempt-1)), + задержка равномерно в [d/2, d]. Половина гарантирует паузу (не долбим упавший S3), + случайная половина разносит повторы многих заданий во времени (нет «стада»).""" + d = min(float(cap), float(base) * (2 ** max(0, attempt - 1))) + return d / 2 + rng.uniform(0, d / 2) + + +def add_event(conn, job_id, event, attempt=None, actor=None, detail=None): + conn.execute( + "INSERT INTO job_events (job_id, event, attempt, actor, detail) VALUES (%s, %s, %s, %s, %s)", + (job_id, event, attempt, actor, json.dumps(detail, ensure_ascii=False) if detail is not None else None)) + + +def _previous_trace(conn, job_id): + row = conn.execute( + """SELECT payload->>'traceparent' AS traceparent, payload->>'tracestate' AS tracestate FROM outbox + WHERE job_id = %s AND payload ? 'traceparent' ORDER BY id DESC LIMIT 1""", (job_id,)).fetchone() + if row is None: + return {} + row = row if isinstance(row, dict) else {"traceparent": row[0], "tracestate": row[1]} + return {k: v for k, v in row.items() if v} + + +def add_outbox(conn, job_id, attachment_id, pipeline_version, attempt, delay_seconds=0.0): + if telemetry.has_active_span(): + attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "create", + "messaging.operation.name": "create", "messaging.destination.name": "attachments", + "messenger.job.id": str(job_id), "messenger.attachment.id": str(attachment_id), + "messenger.job.attempt": int(attempt), "messenger.outbox.delay_s": round(float(delay_seconds), 3)} + with telemetry.child_span(JOB_TYPE + " create", SpanKind.PRODUCER, attrs): + return _insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, delay_seconds, + telemetry.inject_dict()) + return _insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, delay_seconds, + _previous_trace(conn, job_id)) + + +def _insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, delay_seconds, trace_fields): + payload = make_payload(job_id, attachment_id, pipeline_version, attempt, trace_fields=trace_fields) + conn.execute( + """INSERT INTO outbox (job_id, payload, available_at) + VALUES (%s, %s, now() + make_interval(secs => %s))""", + (job_id, json.dumps(payload), delay_seconds)) + return payload + + +# --------------------------------------------------------------------------- +# API: постановка в очередь +# --------------------------------------------------------------------------- +@db_operation("jobs.enqueue") +def enqueue(conn, attachment_id, pipeline_version, max_attempts): + """В ТЕКУЩЕЙ транзакции: pending -> queued, jobs, outbox, событие. job_id или None.""" + row = conn.execute( + """UPDATE attachments SET status = 'queued', updated_at = now() + WHERE id = %s AND status = 'pending' RETURNING id""", (attachment_id,)).fetchone() + if row is None: + return None + job_id = uuid.uuid4() + conn.execute( + """INSERT INTO jobs (id, attachment_id, pipeline_version, status, attempt, max_attempts, + next_attempt_at, created_at, updated_at) + VALUES (%s, %s, %s, 'queued', 0, %s, now(), now(), now())""", + (job_id, attachment_id, pipeline_version, max_attempts)) + add_event(conn, job_id, "enqueued", 1, "api") + add_outbox(conn, job_id, attachment_id, pipeline_version, 1) + return job_id + + +# --------------------------------------------------------------------------- +# Worker +# --------------------------------------------------------------------------- +JOB_COLUMNS = """j.id AS job_id, j.attachment_id, j.pipeline_version, j.status AS job_status, j.attempt, + j.max_attempts, j.next_attempt_at, j.error_code, j.lease_owner, j.lease_token, j.lease_until, j.replay_count, + j.lease_until > now() AS lease_alive, j.next_attempt_at > now() + interval '1 second' AS too_early, + GREATEST(0, EXTRACT(EPOCH FROM now() - j.next_attempt_at))::float8 AS eligible_wait_s, + EXTRACT(EPOCH FROM now() - j.created_at)::float8 AS job_age_s, + a.status AS attachment_status, a.content_type, a.size_bytes, a.sha256, a.width, a.height, + a.object_key, a.staging_key, a.filename, a.pipeline_version AS current_pipeline_version""" + + +def load_job(conn, job_id, lock=False): + return conn.execute( + "SELECT " + JOB_COLUMNS + " FROM jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.id = %s" + + (" FOR UPDATE OF j" if lock else ""), (job_id,)).fetchone() + + +def _owner_host(owner): + return (owner or "").split(":", 1)[0] + + +@db_operation("jobs.claim") +def claim(conn, msg, owner, lease_seconds): + """Решение по сообщению. Возвращает (action, job): + 'process' — аренда взята (job со свежим lease_token), можно работать; + 'ack' — ничего делать не нужно (уже ready/failed, занято живой арендой, ранний дубль); + 'reject' — сообщение не соответствует заданию (в DLQ, разбирает оператор). + Вызывается в транзакции; при 'process' коммит фиксирует processing ДО начала работы.""" + job = load_job(conn, msg["job_id"], lock=True) + if job is None: + return "reject", "unknown_job" + if str(job["attachment_id"]) != msg["attachment_id"] or job["pipeline_version"] != msg["pipeline_version"]: + return "reject", "job_mismatch" + status = job["job_status"] + if status == "ready": + add_event(conn, job["job_id"], "duplicate_ignored", msg["attempt"], owner, {"status": "ready"}) + return "ack", job + if status == "failed": + add_event(conn, job["job_id"], "duplicate_ignored", msg["attempt"], owner, {"status": "failed"}) + return "ack", job + if job["current_pipeline_version"] > job["pipeline_version"]: + # Устаревшая job после новой версии pipeline: результат не публикуем. + conn.execute("""UPDATE jobs SET status = 'failed', error_code = 'superseded', lease_owner = NULL, + lease_token = NULL, lease_until = NULL, updated_at = now(), finished_at = now() + WHERE id = %s""", (job["job_id"],)) + add_event(conn, job["job_id"], "superseded", msg["attempt"], owner) + return "ack", job + if status == "processing" and job["lease_alive"]: + prev = job["lease_owner"] or "" + # Тот же контейнер (hostname), но другой процесс (boot id): прежняя инкарнация этого + # worker умерла (crash/restart) — два процесса одного контейнера одновременно не живут. + if not (_owner_host(prev) == _owner_host(owner) and prev != owner): + add_event(conn, job["job_id"], "busy_ignored", msg["attempt"], owner, {"lease_owner": prev}) + return "ack", job + add_event(conn, job["job_id"], "lease_taken_over", job["attempt"], owner, + {"previous_owner": prev, "reason": "previous process of this container died"}) + elif status == "processing": + add_event(conn, job["job_id"], "lease_taken_over", job["attempt"], owner, + {"previous_owner": job["lease_owner"], "reason": "lease expired"}) + elif job["too_early"]: + # queued, но время следующей попытки ещё не пришло: это старый дубль, а запланированное + # сообщение relay опубликует само (outbox.available_at). + add_event(conn, job["job_id"], "early_duplicate_ignored", msg["attempt"], owner) + return "ack", job + if job["attempt"] >= job["max_attempts"]: + _finish_failed(conn, job, "retry_exhausted", owner, "attempts exhausted before claim") + return "ack", job + token = uuid.uuid4() + conn.execute( + """UPDATE jobs SET status = 'processing', attempt = attempt + 1, lease_owner = %s, lease_token = %s, + lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE id = %s""", (owner, token, lease_seconds, job["job_id"])) + conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = %s", + (job["attachment_id"],)) + job = dict(job, attempt=job["attempt"] + 1, lease_token=token, job_status="processing") + add_event(conn, job["job_id"], "claimed", job["attempt"], owner, + {"redelivered_message_attempt": msg["attempt"]}) + return "process", job + + +@db_operation("jobs.heartbeat") +def heartbeat(conn, job_id, token, lease_seconds): + """Продлить аренду. False — аренда потеряна (reaper/другой worker), работу надо бросить.""" + cur = conn.execute( + """UPDATE jobs SET lease_until = now() + make_interval(secs => %s) + WHERE id = %s AND lease_token = %s AND status = 'processing'""", (lease_seconds, job_id, token)) + return cur.rowcount == 1 + + +@db_operation("jobs.complete") +def complete(conn, job, variants, owner): + """processing -> ready атомарно с метаданными вариантов. Только владелец аренды (fencing по + lease_token): опоздавший процесс, у которого аренду забрали, ничего не перезапишет. + Возвращает 'ready' | 'already_ready' | 'lease_lost'.""" + cur = conn.execute( + """UPDATE jobs SET status = 'ready', lease_owner = NULL, lease_token = NULL, lease_until = NULL, + error_code = NULL, last_error = NULL, updated_at = now(), finished_at = now() + WHERE id = %s AND status = 'processing' AND lease_token = %s + RETURNING EXTRACT(EPOCH FROM clock_timestamp() - created_at)::float8 AS time_to_ready_s""", + (job["job_id"], job["lease_token"])) + row = cur.fetchone() if cur.rowcount == 1 else None + if row is not None: + # От durable acceptance (COMMIT enqueue в API = jobs.created_at) до публикации ready. + job["time_to_ready_s"] = row["time_to_ready_s"] if isinstance(row, dict) else row[0] + if cur.rowcount != 1: + now = load_job(conn, job["job_id"]) + add_event(conn, job["job_id"], "commit_rejected_lease_lost", job["attempt"], owner, + {"current_status": now["job_status"]}) + return "already_ready" if now["job_status"] == "ready" else "lease_lost" + with conn.cursor() as cur: + # Ключ эффекта — PK (attachment_id, pipeline_version, name): второго набора не будет. + cur.executemany( + """INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, + size_bytes, width, height, sha256, created_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) + ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET + object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, + sha256 = EXCLUDED.sha256""", + [(job["attachment_id"], job["pipeline_version"], v["name"], v["key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"]) for v in variants]) + # Не откатываем более новую версию pipeline (на случай нескольких версий, трек 5). + conn.execute( + """UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now() + WHERE id = %s AND pipeline_version <= %s""", (job["attachment_id"], job["pipeline_version"])) + add_event(conn, job["job_id"], "ready", job["attempt"], owner, + {"variants": {v["name"]: [v["width"], v["height"]] for v in variants}}) + return "ready" + + +def _finish_failed(conn, job, code, actor, detail): + row = conn.execute( + """UPDATE jobs SET status = 'failed', error_code = %s, last_error = %s, lease_owner = NULL, + lease_token = NULL, lease_until = NULL, updated_at = now(), finished_at = now() + WHERE id = %s RETURNING EXTRACT(EPOCH FROM clock_timestamp() - created_at)::float8 AS t""", + (code, (detail or "")[:500], job["job_id"])).fetchone() + if row is not None and isinstance(job, dict): + job["time_to_terminal_s"] = row["t"] if isinstance(row, dict) else row[0] + conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() WHERE id = %s", + (code, job["attachment_id"])) + add_event(conn, job["job_id"], "failed", job["attempt"], actor, {"error_code": code, "detail": detail}) + + +@db_operation("jobs.fail_or_retry") +def fail_or_retry(conn, job, code, detail, owner, base, cap, rng=random): + """Ошибка попытки. Постоянная -> failed(code); временная -> queued с backoff, пока есть + попытки, иначе failed(retry_exhausted). Только владелец аренды. + Возвращает 'failed' | 'retry' | 'lease_lost' и задержку.""" + cur = load_job(conn, job["job_id"], lock=True) + if cur["job_status"] != "processing" or cur["lease_token"] != job["lease_token"]: + add_event(conn, job["job_id"], "error_ignored_lease_lost", job["attempt"], owner, {"error_code": code}) + return "lease_lost", None + if code in PERMANENT_ERRORS: + _finish_failed(conn, job, code, owner, detail) + return "failed", None + if cur["attempt"] >= cur["max_attempts"]: + _finish_failed(conn, job, "retry_exhausted", owner, "last error: %s %s" % (code, detail or "")) + return "failed", None + delay = backoff_seconds(cur["attempt"], base, cap, rng) + _schedule_retry(conn, cur, delay, code, detail, owner, "retry_scheduled") + return "retry", delay + + +def _schedule_retry(conn, job, delay, code, detail, actor, event): + conn.execute( + """UPDATE jobs SET status = 'queued', lease_owner = NULL, lease_token = NULL, lease_until = NULL, + error_code = NULL, last_error = %s, next_attempt_at = now() + make_interval(secs => %s), + updated_at = now() + WHERE id = %s""", (("%s: %s" % (code, detail or ""))[:500], delay, job["job_id"])) + conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s", + (job["attachment_id"],)) + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1, delay) + add_event(conn, job["job_id"], event, job["attempt"], actor, + {"error_code": code, "delay_seconds": round(delay, 3), "next_attempt": job["attempt"] + 1}) + + +# --------------------------------------------------------------------------- +# Reaper (в relay): зависшие processing и потерянные сообщения +# --------------------------------------------------------------------------- +@db_operation("jobs.reap_expired_leases") +def reap_expired_leases(conn, base, cap, actor="reaper", limit=100): + """processing с истёкшей арендой (worker умер/завис и не продлевал lease) -> queued + с backoff или failed(retry_exhausted). Возвращает число обработанных.""" + rows = conn.execute( + "SELECT " + JOB_COLUMNS + """ FROM jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'processing' AND j.lease_until < now() + ORDER BY j.lease_until LIMIT %s FOR UPDATE OF j SKIP LOCKED""", (limit,)).fetchall() + for job in rows: + add_event(conn, job["job_id"], "lease_expired", job["attempt"], actor, {"lease_owner": job["lease_owner"]}) + if job["attempt"] >= job["max_attempts"]: + _finish_failed(conn, job, "retry_exhausted", actor, "lease expired on last attempt") + else: + _schedule_retry(conn, job, backoff_seconds(job["attempt"], base, cap), "lease_expired", + "worker did not renew lease", actor, "requeued_after_lease") + return len(rows) + + +@db_operation("jobs.redrive_stale_queued") +def redrive_stale_queued(conn, after_seconds, actor="reaper", limit=100): + """queued-задание, по которому давно нет ни неопубликованного outbox, ни движения: + сообщение могло потеряться в брокере (например, после потери его volume). Публикуем + ещё раз — дубль безопасен, решение примет worker по строке jobs.""" + rows = conn.execute( + """SELECT j.id, j.attachment_id, j.pipeline_version, j.attempt FROM jobs j + WHERE j.status = 'queued' AND j.updated_at < now() - make_interval(secs => %s) + AND j.next_attempt_at < now() - make_interval(secs => %s) + AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.id AND o.published_at IS NULL) + ORDER BY j.updated_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""", + (after_seconds, after_seconds, limit)).fetchall() + for r in rows: + conn.execute("UPDATE jobs SET updated_at = now() WHERE id = %s", (r["id"],)) + add_outbox(conn, r["id"], r["attachment_id"], r["pipeline_version"], r["attempt"] + 1) + add_event(conn, r["id"], "redriven", r["attempt"] + 1, actor, {"stale_after_seconds": after_seconds}) + return len(rows) + + +# --------------------------------------------------------------------------- +# Оператор: replay +# --------------------------------------------------------------------------- +@db_operation("jobs.replay") +def replay(conn, attachment_id, actor, extra_attempts, reason, dry_run=False): + """failed -> queued. Новая попытка того же job_id (прежний ключ эффекта), max_attempts + увеличивается на extra_attempts. ready/queued/processing НЕ трогаются: «повторить» не + означает выполнить дубль успешной работы. Возвращает (result, job).""" + job = conn.execute( + "SELECT " + JOB_COLUMNS + """ FROM jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = %s AND j.pipeline_version = a.pipeline_version FOR UPDATE OF j""", + (attachment_id,)).fetchone() + if job is None: + return "not_found", None + if job["job_status"] != "failed": + return "refused_status_" + job["job_status"], job + # Аудит F01: staging_key перечитывается ОТДЕЛЬНЫМ запросом под блокировкой строки вложения. + # В READ COMMITTED «SELECT ... FOR UPDATE OF j» после ожидания блокировки перечитывает только + # строку jobs, а колонки attachments из соединения могли остаться из старого снимка — и replay + # принял бы работу, исходник которой janitor уже освободил. + att = conn.execute("SELECT staging_key FROM attachments WHERE id = %s FOR UPDATE", + (job["attachment_id"],)).fetchone() + if att["staging_key"] is None: + return "refused_staging_expired", job + if dry_run: + return "would_replay", job + conn.execute( + """UPDATE jobs SET status = 'queued', max_attempts = attempt + %s, error_code = NULL, + next_attempt_at = now(), replay_count = replay_count + 1, updated_at = now(), finished_at = NULL + WHERE id = %s""", (extra_attempts, job["job_id"])) + conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = %s", + (job["attachment_id"],)) + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1) + add_event(conn, job["job_id"], "replayed", job["attempt"] + 1, actor, + {"reason": reason, "previous_error": job.get("error_code"), "replay_count": job["replay_count"] + 1}) + return "replayed", job diff --git a/src/jpeg_probe.py b/src/jpeg_probe.py new file mode 100644 index 0000000..640ef44 --- /dev/null +++ b/src/jpeg_probe.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Пробное декодирование JPEG в ОТДЕЛЬНОМ процессе с лимитами (аудит F02). + + python jpeg_probe.py < файл на stdin (обычный файл, seekable) + +API раньше декодировал JPEG в своём процессе, рассчитывая, что draft(1/8) уменьшает +память в ~64 раза. Для progressive JPEG это неверно: декодеру нужен буфер DCT- +коэффициентов ВСЕГО кадра. Замер на Pillow 13, 6324x6324 (~40 Мпикс): baseline — 23 MiB, +progressive 4:2:0 — 137 MiB, progressive 4:4:4 — 252 MiB на одно декодирование, а лимит +реплики был 256 MiB. Поэтому проверка вынесена в подпроцесс: + * RLIMIT_AS / RLIMIT_CPU ставятся ДО импорта Pillow; + * oom_score_adj=1000: если памяти контейнера всё же не хватит, ядро убьёт пробу, а не API; + * код выхода: 0 — корректный JPEG; 3 — повреждён/не тот формат; 5 — не хватило памяти/CPU. +""" +import os +import resource +import sys + +EXIT_OK, EXIT_INVALID, EXIT_RESOURCE = 0, 3, 5 + + +def main(): + max_pixels, width, height = (int(x) for x in sys.argv[1:4]) + limit = int(os.environ.get("JPEG_PROBE_MEMORY_MB", "384")) * 1024 * 1024 + cpu = int(os.environ.get("JPEG_PROBE_CPU_SECONDS", "10")) + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 1)) + try: + with open("/proc/self/oom_score_adj", "w") as f: + f.write("1000") + except OSError: + pass + import warnings + from PIL import Image + Image.MAX_IMAGE_PIXELS = max_pixels + warnings.simplefilter("error", Image.DecompressionBombWarning) + src = os.fdopen(0, "rb", closefd=False) + try: + with Image.open(src, formats=("JPEG",)) as im: + if im.size != (width, height): + return EXIT_INVALID + im.draft("RGB", (max(1, width // 8), max(1, height // 8))) + im.load() # обрезанные/битые энтропийные данные -> исключение + except MemoryError: + return EXIT_RESOURCE + except Exception: + return EXIT_INVALID + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..a18b977 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Однократное применение миграций и прав (compose-сервис `migrate`). + +Запускается ОДИН раз до старта/замены API-реплик, а не каждой репликой. Миграции — под +pg_advisory_lock: параллельный повторный запуск не устроит гонку; каждый файл в своей транзакции. + +Lab7: + * job подключается как суперпользователь postgres по TLS verify-full; пароль — из OpenBao + (kv/messenger/migrate, cert auth identity `migrate`), не из файла; + * создаёт/обновляет сервисные роли api_svc, worker_svc, relay_svc, janitor_svc, ops_svc + (LOGIN, пароль из того же KV; NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT, + ограничение числа соединений) — повторный запуск синхронизирует пароль (ротация); + * применяет migrations/security/grants.sql (REVOKE ALL -> точные GRANT). +""" +import logging +import sys +from pathlib import Path + +from psycopg import sql + +import telemetry +import vault +from server import MIGRATIONS_DIR, STORE + +log = logging.getLogger("messenger.migrate") + +ROLES = { # роль -> (поле KV с паролем, CONNECTION LIMIT) + "api_svc": ("DB_API_PASSWORD", 40), + "worker_svc": ("DB_WORKER_PASSWORD", 30), + "relay_svc": ("DB_RELAY_PASSWORD", 10), + "janitor_svc": ("DB_JANITOR_PASSWORD", 5), + "ops_svc": ("DB_OPS_PASSWORD", 3), +} + + +def ensure_roles(conn): + for role, (field, limit) in ROLES.items(): + password = vault.secret(field) + exists = conn.execute("SELECT 1 FROM pg_roles WHERE rolname = %s", (role,)).fetchone() + verb = "ALTER" if exists else "CREATE" + conn.execute(sql.SQL("{} ROLE {} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION " + "NOBYPASSRLS CONNECTION LIMIT {} PASSWORD {}").format( + sql.SQL(verb), sql.Identifier(role), sql.Literal(limit), sql.Literal(password))) + # Без лишних параметров сессии, которые могла бы выставить роль сама. + conn.execute(sql.SQL("ALTER ROLE {} SET statement_timeout = '5s'").format(sql.Identifier(role))) + log.info("db role %s %s", role, "updated" if exists else "created", extra={"event": "db.role"}) + + +def main(): + telemetry.configure_logging() # короткоживущий job: только JSON-лог в stdout, без экспорта + STORE.open() + try: + before = STORE.pending_migrations(MIGRATIONS_DIR) + STORE.migrate(MIGRATIONS_DIR) + grants = (Path(MIGRATIONS_DIR) / "security" / "grants.sql").read_text(encoding="utf-8") + with STORE.pool.connection() as conn: + with conn.transaction(): + ensure_roles(conn) + conn.execute(grants) + log.info("migrations done; applied now: %s; roles and grants synced", ", ".join(before) or "nothing") + finally: + STORE.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/objstore.py b/src/objstore.py new file mode 100644 index 0000000..c909598 --- /dev/null +++ b/src/objstore.py @@ -0,0 +1,233 @@ +"""Доступ к S3-совместимому хранилищу (SeaweedFS) через клиент minio-py. + +API работает с bucket под ОТДЕЛЬНЫМИ credentials с правами только Read+Write на этот +bucket (без List, без создания/удаления bucket, без политик и IAM) — см. +scripts/init-secrets.sh и REPORT.md. Публичных URL/redirect нет: байты всегда идут +через авторизованный API. + +Lab6: каждый запрос к S3 — CLIENT span «s3 <Операция>» (внутри запроса/задания) и гистограмма +messenger.s3.operation.duration{messenger.s3.operation, outcome}. В span — bucket и ключ объекта +(ключи назначает сервер: staging/, attachments//v1/; filename в ключ не входит), +без credentials и подписанных URL. +""" +import contextlib +import logging +import os +import threading +import time +from urllib.parse import urlsplit + +import urllib3 +from minio import Minio +from minio.error import S3Error + +import telemetry +import tlsutil +import vault +from telemetry import SpanKind + +log = logging.getLogger("messenger.s3") + +# minio-py для объектов больше part_size делает multipart и держит в памяти +# один part. 5 MiB — минимум S3: файл 10 MiB -> 2 части по 5 MiB. +PART_SIZE = 5 * 1024 * 1024 +STREAM_CHUNK = 64 * 1024 +# Служебные объекты S3Health: health/<реплика> (вне prefix originals/, janitor их не трогает). +HEALTH_PREFIX = "health/" + + +class StorageUnavailable(Exception): + """S3 недоступен/таймаут/ошибка сервера -> 503.""" + + +class ObjectMissing(Exception): + """Метаданные есть, объекта нет (рассогласование) -> 503 + лог.""" + + +def _secret(name): + # S3_CLIENT_REQUIRED=0 — только job migrate (импортирует server.py, к S3 не обращается). + return vault.secret(name, required=os.environ.get("S3_CLIENT_REQUIRED", "1") == "1") + + +def make_client(access_env="S3_ACCESS_KEY_ID", secret_env="S3_SECRET_ACCESS_KEY", + connect_timeout=None, read_timeout=None): + endpoint = urlsplit(os.environ.get("S3_ENDPOINT", "https://s3:8333")) + connect_timeout = connect_timeout or float(os.environ.get("S3_CONNECT_TIMEOUT_SECONDS", "2")) + read_timeout = read_timeout or float(os.environ.get("S3_READ_TIMEOUT_SECONDS", "5")) + # Короткие таймауты и без собственных повторов: зависший S3 не держит поток + # дольше таймаута LB, повтор решает клиент (для upload — новым запросом). + # Lab7: HTTPS к S3 с проверкой CA и hostname; plaintext endpoint запрещён. + if endpoint.scheme != "https": + raise SystemExit("S3_ENDPOINT must be https:// in lab7") + http = urllib3.PoolManager( + cert_reqs="CERT_REQUIRED", ca_certs=tlsutil.CA_FILE, + timeout=urllib3.Timeout(connect=connect_timeout, read=read_timeout), + retries=urllib3.Retry(total=0, connect=0, read=0, redirect=0, status=0), + maxsize=int(os.environ.get("S3_POOL_SIZE", "16")), + block=False) + return Minio(endpoint.netloc, access_key=_secret(access_env), secret_key=_secret(secret_env), + secure=endpoint.scheme == "https", region=os.environ.get("S3_REGION", "us-east-1"), + http_client=http) + + +def _is_transport_error(exc): + return isinstance(exc, (urllib3.exceptions.HTTPError, OSError)) + + +@contextlib.contextmanager +def _s3_op(operation, bucket, key): + started = time.perf_counter() + outcome = "ok" + attrs = {"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": operation, + "aws.s3.bucket": bucket, "server.address": "s3"} + if key and not key.startswith(HEALTH_PREFIX): + attrs["aws.s3.key"] = key + try: + with telemetry.child_span("s3 " + operation, SpanKind.CLIENT, attrs): + yield + except ObjectMissing: + outcome = "not_found" + raise + except BaseException: + outcome = "error" + raise + finally: + telemetry.s3_duration.record(time.perf_counter() - started, + {"messenger.s3.operation": operation, "outcome": outcome}) + + +class ObjectStore: + def __init__(self, client, bucket): + self.client = client + self.bucket = bucket + + def put(self, key, fileobj, length, content_type): + with _s3_op("PutObject", self.bucket, key): + self._put(key, fileobj, length, content_type) + + def _put(self, key, fileobj, length, content_type): + try: + self.client.put_object(self.bucket, key, fileobj, length, content_type=content_type, + part_size=PART_SIZE) + except S3Error as exc: + log.warning("s3 put failed: %s", exc.code) + raise StorageUnavailable() + except Exception as exc: + if _is_transport_error(exc): + log.warning("s3 put failed: %s", type(exc).__name__) + raise StorageUnavailable() + raise + + def open(self, key): + """Поток объекта (urllib3 response). Закрывать: resp.close(); resp.release_conn().""" + with _s3_op("GetObject", self.bucket, key): + return self._open(key) + + def _open(self, key): + try: + return self.client.get_object(self.bucket, key) + except S3Error as exc: + if exc.code in ("NoSuchKey", "NoSuchObject"): + raise ObjectMissing() + log.warning("s3 get failed: %s", exc.code) + raise StorageUnavailable() + except Exception as exc: + if _is_transport_error(exc): + log.warning("s3 get failed: %s", type(exc).__name__) + raise StorageUnavailable() + raise + + def delete(self, key): + """Идемпотентно: отсутствие объекта — не ошибка (S3 DELETE так и работает).""" + with _s3_op("DeleteObject", self.bucket, key): + self._delete(key) + + def _delete(self, key): + try: + self.client.remove_object(self.bucket, key) + except S3Error as exc: + if exc.code in ("NoSuchKey", "NoSuchObject"): + return + raise StorageUnavailable() + except Exception as exc: + if _is_transport_error(exc): + raise StorageUnavailable() + raise + + def ping(self, probe_key): + """PUT + GET маленького служебного объекта health/<реплика>: проверяет S3, + credentials и РЕАЛЬНЫЕ пути записи и чтения. HEAD bucket/healthz для этого мало: + у SeaweedFS после рестарта они успешны, а первая запись ещё ~5 с висит + (выбор raft-лидера master, выдача volume под запись).""" + import io + with _s3_op("HealthProbe", self.bucket, None): + return self._ping(probe_key, io) + + def _ping(self, probe_key, io): + self.client.put_object(self.bucket, probe_key, io.BytesIO(b"ok"), 2, content_type="text/plain") + resp = self.client.get_object(self.bucket, probe_key) + try: + if resp.read() != b"ok": + raise RuntimeError("probe mismatch") + finally: + resp.close() + resp.release_conn() + return True + + +class S3Health: + """Фоновая проверка S3, как DbHealth для PostgreSQL. + + S3 НЕ входит в /health/ready: он общий для всех реплик, и его отказ не лечится + исключением реплики из LB, а текстовые сообщения без него работают. Флаг нужен, + чтобы upload при известном отказе S3 сразу получал 503, не принимая 10 MiB.""" + + def __init__(self, store, interval, stale_after, instance_id): + self.store = store + self.probe_key = HEALTH_PREFIX + instance_id + self.interval = interval + self.stale_after = stale_after + self.last_ok = None + self._stop = threading.Event() + self._ok_event = threading.Event() # выставлен, пока последняя проверка успешна + + def start(self): + threading.Thread(target=self._loop, name="s3-health", daemon=True).start() + + def stop(self): + self._stop.set() + + def ok(self): + last = self.last_ok + return last is not None and time.monotonic() - last <= self.stale_after + + def wait_startup(self, timeout): + """Только на старте реплики (успешной проверки ещё не было): подождать до timeout, + пока S3 станет доступен для записи. При подтверждённом отказе — не ждать.""" + if self.last_ok is None: + self._ok_event.wait(timeout) + return self.ok() + + def known_down(self): + """S3 был доступен и перестал отвечать. На старте реплики (успешной проверки ещё + не было) — False: чтение может работать раньше, чем прогреется запись.""" + last = self.last_ok + return last is not None and time.monotonic() - last > self.stale_after + + def _loop(self): + healthy = None + while not self._stop.is_set(): + try: + self.store.ping(self.probe_key) + self.last_ok = time.monotonic() + self._ok_event.set() + if healthy is not True: + log.info("s3 health: ok", extra={"event": "s3.health.ok"}) + healthy = True + except Exception as exc: + self._ok_event.clear() + if healthy is not False: + log.warning("s3 health: failing (%s)", type(exc).__name__, + extra={"event": "s3.health.failing", "error.type": type(exc).__name__}) + healthy = False + self._stop.wait(self.interval) diff --git a/src/ops.py b/src/ops.py new file mode 100644 index 0000000..38c2b91 --- /dev/null +++ b/src/ops.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Операторский CLI (compose-сервис `ops`, profile tools). Вывод — JSON без секретов. + + ops inspect состояние: attachment, job, события, варианты, outbox, объекты S3 + ops failed [--limit N] реестр failed-заданий (DLQ на стороне БД) + ops replay ... [--all-failed] [--error-code C] [--dry-run] [--reason TEXT] + явный replay failed -> queued (ready/queued/processing не трогаются) + ops dlq [--replay] [--purge-invalid] + сообщения в RabbitMQ DLQ: показать; --replay — разобрать каждое + по состоянию jobs (failed -> replay, остальное -> отбросить с audit) + ops publish-raw учебное: опубликовать произвольное тело (проверка DLQ) + +Replay всегда проходит через строку jobs под FOR UPDATE: повтор ready-задания отклоняется, +поэтому «повторить» не означает исполнить дубль уже успешной работы. +""" +import argparse +import json +import os +import sys + +import pika +import psycopg +from psycopg.rows import dict_row + +import common +import telemetry +import jobs +import objstore + +EXTRA_ATTEMPTS = common.env_int("JOB_MAX_ATTEMPTS", 5) + + +def db(): + return psycopg.connect(common.database_conninfo(), row_factory=dict_row) + + +def jsonable(value): + if isinstance(value, dict): + return {k: jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [jsonable(v) for v in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if hasattr(value, "isoformat"): + return value.isoformat().replace("+00:00", "Z") + return str(value) + + +def out(data): + print(json.dumps(jsonable(data), ensure_ascii=False, indent=1)) + + +def s3_objects(prefix): + """Только имена/размеры (ключ janitor: List, без чтения содержимого).""" + store = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments")) + try: + return sorted((o.object_name, o.size) for o in store.client.list_objects(store.bucket, prefix=prefix, recursive=True)) + except Exception as exc: + return [("error", type(exc).__name__)] + + +def cmd_inspect(args): + with db() as conn: + att = conn.execute( + """SELECT id, chat_id, status, error_code, content_type, size_bytes, sha256, width, height, + object_key, staging_key, pipeline_version, created_at, updated_at + FROM attachments WHERE id = %s""", (args.attachment_id,)).fetchone() + if att is None: + out({"error": "not_found"}) + return 1 + job = conn.execute( + """SELECT id, pipeline_version, status, attempt, max_attempts, next_attempt_at, lease_owner, + lease_until, error_code, last_error, replay_count, created_at, updated_at, finished_at + FROM jobs WHERE attachment_id = %s ORDER BY pipeline_version""", (args.attachment_id,)).fetchall() + job_ids = [j["id"] for j in job] + events = conn.execute( + "SELECT job_id, at, event, attempt, actor, detail FROM job_events WHERE job_id = ANY(%s) ORDER BY id", + (job_ids,)).fetchall() + outbox = conn.execute( + """SELECT id, payload->>'attempt' AS attempt, available_at, created_at, published_at, publish_count, + payload->>'traceparent' AS traceparent + FROM outbox WHERE job_id = ANY(%s) ORDER BY id""", (job_ids,)).fetchall() + variants = conn.execute( + """SELECT pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256 + FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name""", + (args.attachment_id,)).fetchall() + objects = s3_objects("attachments/%s/" % args.attachment_id) + staging = s3_objects(att["staging_key"]) if att["staging_key"] else [] + out({"attachment": att, "jobs": job, "events": events, "outbox": outbox, "variants": variants, + "s3_final_objects": objects, "s3_staging_objects": staging}) + return 0 + + +def cmd_failed(args): + with db() as conn: + rows = conn.execute( + """SELECT j.attachment_id, j.id AS job_id, j.error_code, j.attempt, j.max_attempts, j.replay_count, + j.last_error, j.updated_at, a.staging_key IS NOT NULL AS replayable + FROM jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT %s""", (args.limit,)).fetchall() + out({"failed": rows, "count": len(rows)}) + return 0 + + +def cmd_replay(args): + results = [] + with db() as conn: + ids = list(args.attachment_ids) + if args.all_failed: + q = "SELECT attachment_id FROM jobs WHERE status = 'failed'" + params = [] + if args.error_code: + q += " AND error_code = %s" + params.append(args.error_code) + ids += [str(r["attachment_id"]) for r in conn.execute(q, params)] + for att_id in ids: + with conn.transaction(): + result, job = jobs.replay(conn, att_id, "ops:" + common.HOSTNAME, EXTRA_ATTEMPTS, + args.reason, dry_run=args.dry_run) + results.append({"attachment_id": att_id, "result": result, + "job_id": str(job["job_id"]) if job else None}) + out({"replay": results}) + return 0 if all(r["result"] in ("replayed", "would_replay") for r in results) else 3 + + +def broker_channel(): + conn = pika.BlockingConnection(common.broker_parameters()) + return conn, conn.channel() + + +def cmd_dlq(args): + conn, ch = broker_channel() + seen = [] + tags = [] + try: + with db() as dbc: + while True: + method, props, body = ch.basic_get(common.DLQ, auto_ack=False) + if method is None: + break + deaths = (props.headers or {}).get("x-death") or [] + reason = deaths[0].get("reason") if deaths else None + item = {"reason": reason.decode() if isinstance(reason, bytes) else reason, + "delivery_count": (props.headers or {}).get("x-delivery-count"), "action": "kept"} + try: + msg = jobs.parse_message(body) + item["job_id"] = msg["job_id"] + if args.replay: + with dbc.transaction(): + job = jobs.load_job(dbc, msg["job_id"], lock=True) + if job is None: + item["action"] = "kept_unknown_job" + elif job["job_status"] == "failed": + result, _ = jobs.replay(dbc, str(job["attachment_id"]), "ops-dlq", EXTRA_ATTEMPTS, + "dlq replay") + item["action"] = result + else: + # queued/processing/ready: состояние уже ведёт БД, дубль не нужен. + jobs.add_event(dbc, job["job_id"], "dlq_message_dropped", msg["attempt"], "ops-dlq", + {"job_status": job["job_status"], "reason": item["reason"]}) + item["action"] = "dropped_job_" + job["job_status"] + except jobs.InvalidMessage as exc: + item["invalid"] = str(exc) + if args.purge_invalid: + item["action"] = "purged_invalid" + seen.append(item) + if item["action"].startswith("kept"): + tags.append(method.delivery_tag) # вернём в DLQ после обхода + else: + ch.basic_ack(method.delivery_tag) + for tag in tags: + ch.basic_nack(tag, requeue=True) + finally: + conn.close() + out({"dlq_messages": seen, "count": len(seen)}) + return 0 + + +def cmd_publish_raw(args): + conn, ch = broker_channel() + try: + ch.confirm_delivery() + ch.basic_publish(common.EXCHANGE, common.ROUTING_KEY, args.body.encode(), + pika.BasicProperties(delivery_mode=2, content_type="application/json"), mandatory=True) + finally: + conn.close() + out({"published": True}) + return 0 + + +def main(): + telemetry.configure_logging(stream=sys.stderr) # stdout — только JSON-результат команды + p = argparse.ArgumentParser(prog="ops") + sub = p.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("inspect") + s.add_argument("attachment_id") + s.set_defaults(func=cmd_inspect) + s = sub.add_parser("failed") + s.add_argument("--limit", type=int, default=50) + s.set_defaults(func=cmd_failed) + s = sub.add_parser("replay") + s.add_argument("attachment_ids", nargs="*") + s.add_argument("--all-failed", action="store_true") + s.add_argument("--error-code") + s.add_argument("--dry-run", action="store_true") + s.add_argument("--reason", default="operator replay") + s.set_defaults(func=cmd_replay) + s = sub.add_parser("dlq") + s.add_argument("--replay", action="store_true") + s.add_argument("--purge-invalid", action="store_true") + s.set_defaults(func=cmd_dlq) + s = sub.add_parser("publish-raw") + s.add_argument("body") + s.set_defaults(func=cmd_publish_raw) + args = p.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/relay.py b/src/relay.py new file mode 100644 index 0000000..6bc5963 --- /dev/null +++ b/src/relay.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""Outbox relay + reaper (compose-сервис `relay`). + +Закрывает разрыв PostgreSQL <-> broker (transactional outbox): + API в ОДНОЙ транзакции пишет queued-вложение, jobs и outbox. Relay берёт неопубликованные + строки (FOR UPDATE SKIP LOCKED — несколько relay не мешают друг другу), публикует каждую + в RabbitMQ с publisher confirm (persistent, mandatory) и только ПОСЛЕ подтверждения + брокера ставит published_at в той же транзакции. Крах между COMMIT API и publish, между + publish и confirm или между confirm и UPDATE outbox -> строка остаётся неопубликованной и + будет опубликована снова: возможен дубль сообщения (at-least-once), его гасит worker по + состоянию jobs. + +Будит relay NOTIFY из триггера outbox; без уведомления — опрос раз в RELAY_POLL_SECONDS. +Строки с available_at в будущем (retry с backoff) публикуются, когда наступит срок. + +Reaper (раз в REAPER_INTERVAL_SECONDS, под advisory-блокировкой — один на всех): + * processing с истёкшей арендой -> queued (backoff) или failed(retry_exhausted); + * queued без движения дольше JOB_REDRIVE_AFTER_SECONDS и без неопубликованного outbox + -> ещё одна публикация (сообщение могло потеряться вместе с volume брокера); + * опубликованные строки outbox старше OUTBOX_RETENTION_SECONDS удаляются. + +Учебная инъекция отказа: RELAY_FAULT=crash-before-publish | crash-after-publish — +один раз за жизнь контейнера (маркер в /tmp) процесс делает os._exit посреди батча. + +Lab6: + * каждое сообщение публикуется в PRODUCER span «attachments send» — child контекста из + payload.traceparent (span «create» API/worker). Контекст send записывается и в тело + (traceparent/tracestate, по jobs.schema.json), и в AMQP headers — worker продолжает трассу; + * метрики: messenger.outbox.published{outcome}, messenger.outbox.publish_delay (s, от + available_at до publisher confirm), и наблюдаемые gauges очереди по источнику истины + (PostgreSQL): messenger.jobs.count{status}, messenger.jobs.oldest_age{status} (s, возраст + старейшего незавершённого job от durable acceptance), messenger.jobs.ready_lag (s, сколько + ждёт самый старый job, которому уже пора выполняться), messenger.outbox.pending и + messenger.outbox.oldest_age (не опубликовано relay). Gauges считает отдельное соединение + со statement_timeout 2 с раз в интервал экспорта метрик. +""" +import json +import logging +import os +import sys +import threading +import time + +import pika +import psycopg +from psycopg.rows import dict_row + +import common +import jobs +import telemetry +from telemetry import SpanKind, Status, StatusCode + +log = logging.getLogger("relay") +POLL_SECONDS = common.env_float("RELAY_POLL_SECONDS", 1) +BATCH = common.env_int("RELAY_BATCH", 100) +REAPER_INTERVAL = common.env_float("REAPER_INTERVAL_SECONDS", 5) +REDRIVE_AFTER = common.env_int("JOB_REDRIVE_AFTER_SECONDS", 900) +OUTBOX_RETENTION = common.env_int("OUTBOX_RETENTION_SECONDS", 86400) +RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2) +RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60) +FAULT = os.environ.get("RELAY_FAULT", "") +FAULT_MARKER = "/tmp/relay-fault.done" +HEARTBEAT = "/tmp/relay.heartbeat" +REAPER_LOCK_ID = 727003 + +PUBLISHED = telemetry.meter.create_counter( + "messenger.outbox.published", unit="{message}", description="Outbox rows published to the broker") +PUBLISH_DELAY = telemetry.meter.create_histogram( + "messenger.outbox.publish_delay", unit="s", + description="Delay from outbox available_at to broker publisher confirm") + + +class QueueGauges: + """Глубина очереди и возраст заданий по PostgreSQL (источник истины), не по брокеру.""" + QUERY = """ + SELECT + (SELECT json_object_agg(status, n) FROM (SELECT status, count(*) AS n FROM jobs GROUP BY status) c) AS counts, + (SELECT json_object_agg(status, age) FROM ( + SELECT status, EXTRACT(EPOCH FROM now() - min(created_at))::float8 AS age FROM jobs + WHERE status IN ('queued', 'processing') GROUP BY status) a) AS ages, + (SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(next_attempt_at)), 0)::float8 FROM jobs + WHERE status = 'queued' AND next_attempt_at <= now()) AS ready_lag, + (SELECT count(*) FROM outbox WHERE published_at IS NULL AND available_at <= now()) AS outbox_pending, + (SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(available_at)), 0)::float8 FROM outbox + WHERE published_at IS NULL AND available_at <= now()) AS outbox_age""" + STATUSES = ("queued", "processing", "ready", "failed") + + def __init__(self, conninfo): + self.conninfo = conninfo + self.conn = None + self.lock = threading.Lock() + self.cache = (0.0, None) + + def snapshot(self): + # Несколько callbacks за один сбор метрик -> один SQL-запрос (кэш на 2 с). + with self.lock: + at, data = self.cache + if data is not None and time.monotonic() - at < 2: + return data + try: + if self.conn is None or self.conn.closed: + self.conn = psycopg.connect(self.conninfo, autocommit=True, row_factory=dict_row, + options="-c statement_timeout=2000") + data = self.conn.execute(self.QUERY).fetchone() + except psycopg.Error: + try: + if self.conn is not None: + self.conn.close() + except Exception: + pass + self.conn, data = None, None + self.cache = (time.monotonic(), data) + return data + + def register(self): + def counts(options): + d = self.snapshot() + if d is not None: + got = d["counts"] or {} + for st in self.STATUSES: + yield telemetry.metrics.Observation(int(got.get(st, 0)), {"messenger.job.status": st}) + + def ages(options): + d = self.snapshot() + if d is not None: + got = d["ages"] or {} + for st in ("queued", "processing"): + yield telemetry.metrics.Observation(float(got.get(st) or 0.0), {"messenger.job.status": st}) + + def ready_lag(options): + d = self.snapshot() + if d is not None: + yield telemetry.metrics.Observation(float(d["ready_lag"] or 0.0)) + + def outbox_pending(options): + d = self.snapshot() + if d is not None: + yield telemetry.metrics.Observation(int(d["outbox_pending"] or 0)) + + def outbox_age(options): + d = self.snapshot() + if d is not None: + yield telemetry.metrics.Observation(float(d["outbox_age"] or 0.0)) + + m = telemetry.meter + m.create_observable_gauge("messenger.jobs.count", [counts], unit="{job}", + description="Jobs by status (PostgreSQL jobs table)") + m.create_observable_gauge("messenger.jobs.oldest_age", [ages], unit="s", + description="Age of the oldest unfinished job since durable acceptance") + m.create_observable_gauge("messenger.jobs.ready_lag", [ready_lag], unit="s", + description="How long the oldest due queued job has been waiting for a worker") + m.create_observable_gauge("messenger.outbox.pending", [outbox_pending], unit="{message}", + description="Due outbox rows not yet published to the broker") + m.create_observable_gauge("messenger.outbox.oldest_age", [outbox_age], unit="s", + description="Age of the oldest due unpublished outbox row") + + +def fault(point): + if FAULT != point or os.path.exists(FAULT_MARKER): + return + with open(FAULT_MARKER, "w") as f: + f.write(point) + log.warning("FAULT INJECTION %s: exiting without finishing the batch", point, + extra={"event": "fault.injected", "fault.point": point}) + os._exit(86) + + +class Relay: + def __init__(self): + self.conninfo = common.database_conninfo() + self.db = None + self.listen = None + self.broker = None + self.channel = None + self.next_reap = 0.0 + self.stop = threading.Event() + + # --- соединения --- + def ensure_db(self): + if self.db is None or self.db.closed: + self.db = psycopg.connect(self.conninfo, row_factory=dict_row) + if self.listen is None or self.listen.closed: + self.listen = psycopg.connect(self.conninfo, autocommit=True) + self.listen.execute("LISTEN outbox") + + def ensure_broker(self): + if self.broker is None or self.broker.is_closed or self.channel is None or self.channel.is_closed: + self.close_broker() + self.broker = common.connect_broker(log, self.stop) + if self.broker is None: + return + self.channel = self.broker.channel() + # Publisher confirms: basic_publish ждёт ack брокера (сообщение записано в + # quorum-очередь) и бросает NackError/UnroutableError при отказе. + self.channel.confirm_delivery() + log.info("connected to broker", extra={"event": "broker.connected"}) + + def close_broker(self): + try: + if self.broker is not None and not self.broker.is_closed: + self.broker.close() + except Exception: + pass + self.broker = self.channel = None + + def close_db(self): + for conn in (self.db, self.listen): + try: + if conn is not None: + conn.close() + except Exception: + pass + self.db = self.listen = None + + # --- публикация --- + def publish_batch(self): + """Один батч в одной транзакции. Возвращает число опубликованных строк.""" + spans = [] + try: + with self.db.transaction(): + rows = self.db.execute( + """SELECT id, job_id, payload, EXTRACT(EPOCH FROM now() - available_at)::float8 AS delay_s + FROM outbox WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", (BATCH,)).fetchall() + if not rows: + return 0 + fault("crash-before-publish") # COMMIT задания есть, publish ещё нет + for row in rows: + payload = dict(row["payload"]) + parent = telemetry.extract({k: payload[k] for k in ("traceparent", "tracestate") if k in payload}) + span = telemetry.tracer.start_span( + "attachments send", kind=SpanKind.PRODUCER, context=parent, + attributes={"messaging.system": "rabbitmq", "messaging.operation.type": "send", + "messaging.operation.name": "publish", + "messaging.destination.name": common.EXCHANGE, + "messaging.rabbitmq.destination.routing_key": common.ROUTING_KEY, + "messaging.message.id": "%s:%s" % (payload["job_id"], payload["attempt"]), + "messenger.job.id": payload["job_id"], + "messenger.attachment.id": payload["attachment_id"], + "messenger.job.attempt": payload["attempt"], + "messenger.outbox.id": row["id"], + "messenger.outbox.publish_delay_s": round(row["delay_s"] or 0.0, 3)}) + spans.append((span, row["delay_s"] or 0.0)) + with telemetry.trace.use_span(span, end_on_exit=False): + carrier = telemetry.inject_dict() + payload.pop("tracestate", None) + payload.update(carrier) + self.channel.basic_publish( + exchange=common.EXCHANGE, routing_key=common.ROUTING_KEY, + body=json.dumps(payload, separators=(",", ":")).encode(), + properties=pika.BasicProperties( + content_type="application/json", delivery_mode=pika.DeliveryMode.Persistent, + message_id="%s:%s" % (payload["job_id"], payload["attempt"]), type=jobs.JOB_TYPE, + headers=dict({"x-outbox-id": row["id"]}, **carrier)), + mandatory=True) + fault("crash-after-publish") # confirm получен, published_at не записан -> дубль + self.db.execute( + "UPDATE outbox SET published_at = now(), publish_count = publish_count + 1 WHERE id = ANY(%s)", + ([r["id"] for r in rows],)) + except BaseException as exc: + # Транзакция откатилась: строки останутся неопубликованными и уйдут следующим батчем. + for span, _delay in spans: + span.set_attribute("error.type", type(exc).__name__) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + span.end() + if spans: + PUBLISHED.add(len(spans), {"outcome": "failed", "error.type": type(exc).__name__}) + raise + for span, delay in spans: + PUBLISH_DELAY.record(delay) + span.end() + PUBLISHED.add(len(rows), {"outcome": "ok"}) + log.info("published %d job message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)}) + return len(rows) + + def reap(self): + with self.db.transaction(): + if not self.db.execute("SELECT pg_try_advisory_xact_lock(%s) AS ok", (REAPER_LOCK_ID,)).fetchone()["ok"]: + return + expired = jobs.reap_expired_leases(self.db, RETRY_BASE, RETRY_MAX, actor="reaper:" + common.OWNER) + redriven = jobs.redrive_stale_queued(self.db, REDRIVE_AFTER, actor="reaper:" + common.OWNER) + purged = self.db.execute( + "DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)", + (OUTBOX_RETENTION,)).rowcount + if expired or redriven or purged: + log.info("reaper: expired_leases=%d redriven=%d outbox_purged=%d", expired, redriven, purged, + extra={"event": "reaper.pass", "expired_leases": expired, "redriven": redriven, + "outbox_purged": purged}) + + def wait_notify(self, timeout): + """Ждём NOTIFY outbox не дольше timeout; параллельно обслуживаем heartbeat брокера.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self.broker is not None: + self.broker.process_data_events(time_limit=0) + for _ in self.listen.notifies(timeout=min(0.25, max(0.01, deadline - time.monotonic())), stop_after=1): + return + + def run(self): + log.info("relay started owner=%s fault=%s", common.OWNER, FAULT or "-", extra={"event": "relay.started"}) + delay = 0.5 + while not self.stop.is_set(): + common.touch(HEARTBEAT) + try: + self.ensure_db() + self.ensure_broker() + if self.broker is None: + continue + while self.publish_batch() == BATCH: + pass + if time.monotonic() >= self.next_reap: + self.reap() + self.next_reap = time.monotonic() + REAPER_INTERVAL + self.wait_notify(POLL_SECONDS) + delay = 0.5 + except (pika.exceptions.AMQPError, pika.exceptions.UnroutableError, pika.exceptions.NackError) as exc: + # Транзакция батча откатилась: строки остались неопубликованными. + log.warning("broker publish failed (%s), retry in %.1fs", type(exc).__name__, delay, + extra={"event": "broker.publish_failed", "error.type": type(exc).__name__}) + self.close_broker() + self.stop.wait(delay) + delay = min(5.0, delay * 2) + except psycopg.Error as exc: + log.warning("database error (%s), retry in %.1fs", type(exc).__name__, delay, + extra={"event": "db.error", "error.type": type(exc).__name__}) + self.close_db() + self.stop.wait(delay) + delay = min(5.0, delay * 2) + self.close_broker() + self.close_db() + + +def main(): + import signal + telemetry.setup("messenger-relay") + relay = Relay() + QueueGauges(relay.conninfo).register() + signal.signal(signal.SIGTERM, lambda *_: relay.stop.set()) + signal.signal(signal.SIGINT, lambda *_: relay.stop.set()) + relay.run() + log.info("relay stopped", extra={"event": "relay.stopped"}) + telemetry.shutdown() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 0000000..a572514 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Идемпотентный bootstrap S3 (однократный compose-job `s3-init`). + +Работает под admin-credentials, которые есть ТОЛЬКО у этого job (API и janitor их не +получают). Повторный запуск ничего не ломает: + 1. ждёт доступности S3 (до 60 с); + 2. создаёт bucket, если его нет (уже существующий — не ошибка); + 3. проверяет приватность: анонимные GET объекта и ListBucket должны получить 403. + Если bucket вдруг публичен — exit 1, и API не стартует (depends_on). +Права пользователей (api: Read+Write, janitor: List+Write) задаются декларативно в +secrets/s3_identities.json (генерирует scripts/init-secrets.sh). +""" +import io +import json +import logging +import os +import sys +import time +import urllib.error +import urllib.request + +from minio.error import S3Error + +import objstore +import tlsutil +import telemetry + +telemetry.configure_logging() +log = logging.getLogger("messenger.s3-init") + + +def anonymous_status(url): + try: + # Lab7: анонимная проверка тоже по HTTPS с проверкой CA (без client cert и без ключей). + with urllib.request.urlopen(url, timeout=3, context=tlsutil.client_context(with_identity=False)) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main(): + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + # Первый PUT в новый bucket ждёт, пока SeaweedFS выделит volume-ы коллекции: таймаут больше. + client = objstore.make_client(read_timeout=30) + deadline = time.monotonic() + 60 + while True: + try: + exists = client.bucket_exists(bucket) + break + except Exception as exc: + if time.monotonic() > deadline: + raise + log.info("s3 not ready (%s), retrying", type(exc).__name__) + time.sleep(1) + created = False + if not exists: + try: + client.make_bucket(bucket) + created = True + except S3Error as exc: + if exc.code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise + # Контрольный объект: на нём проверяем, что анонимный GET не проходит. + probe = "bootstrap/probe" + for attempt in range(1, 6): + try: + client.put_object(bucket, probe, io.BytesIO(b"private"), 7, content_type="text/plain") + break + except Exception as exc: + if attempt == 5: + raise + log.info("probe put failed (%s), retry %d", type(exc).__name__, attempt) + time.sleep(2) + checks = { + "anonymous_get_object": anonymous_status("%s/%s/%s" % (endpoint, bucket, probe)), + "anonymous_list_bucket": anonymous_status("%s/%s?list-type=2" % (endpoint, bucket)), + } + client.remove_object(bucket, probe) + private = all(code in (401, 403) for code in checks.values()) + print(json.dumps({"bucket": bucket, "created": created, "already_existed": exists, + "anonymous_http_status": checks, "private": private}), flush=True) + return 0 if private else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..4126ed7 --- /dev/null +++ b/src/security.py @@ -0,0 +1,81 @@ +"""Lab7: пароли — Argon2id (argon2-cffi), сессионные токены — случайные 256 бит. + +Формат хеша — стандартная PHC-строка `$argon2id$v=19$m=,t=,p=$$`: +соль (16 байт, случайная на каждый хеш) и параметры хранятся вместе с хешем, поэтому смена +параметров не ломает старые записи. Параметры по умолчанию — базовый профиль OWASP +(m=19 MiB, t=2, p=1); выбор обоснован замером на стенде (student-tests/argon2_bench.py, +evidence/argon2-bench.json) и задаётся ARGON2_MEMORY_KIB / ARGON2_ITERATIONS / ARGON2_PARALLELISM. + +Rehash: при успешном входе хеш со старыми параметрами или старой схемой (scrypt из lab2–6) +пересчитывается Argon2id с текущими параметрами — plaintext для этого есть только в момент входа. +""" +import base64 +import hashlib +import hmac +import os +import secrets + +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +ARGON2_MEMORY_KIB = int(os.environ.get("ARGON2_MEMORY_KIB", "19456")) +ARGON2_ITERATIONS = int(os.environ.get("ARGON2_ITERATIONS", "2")) +ARGON2_PARALLELISM = int(os.environ.get("ARGON2_PARALLELISM", "1")) +# Нижняя граница задания: не ниже 19 MiB, t=2, p=1 — более слабый профиль процесс не примет. +if ARGON2_MEMORY_KIB < 19456 or ARGON2_ITERATIONS < 2 or ARGON2_PARALLELISM < 1: + raise SystemExit("Argon2id parameters below the lab7 baseline (m>=19456 KiB, t>=2, p>=1)") + +HASHER = PasswordHasher(time_cost=ARGON2_ITERATIONS, memory_cost=ARGON2_MEMORY_KIB, + parallelism=ARGON2_PARALLELISM, hash_len=32, salt_len=16, type=Type.ID) + +# Устаревшая схема lab2–6 (только проверка + rehash, новые хеши не создаются). +_SCRYPT_MAXMEM = 128 * 1024 * 1024 + + +def params(): + return {"algorithm": "argon2id", "memory_kib": ARGON2_MEMORY_KIB, "iterations": ARGON2_ITERATIONS, + "parallelism": ARGON2_PARALLELISM, "salt_bytes": 16, "hash_bytes": 32} + + +def hash_password(password): + return HASHER.hash(password) + + +def _verify_scrypt(password, stored): + try: + scheme, log_n, r, p, salt, digest = stored.split("$") + if scheme != "scrypt": + return False + actual = hashlib.scrypt(password.encode("utf-8"), salt=base64.b64decode(salt), n=2 ** int(log_n), + r=int(r), p=int(p), maxmem=_SCRYPT_MAXMEM, dklen=32) + return hmac.compare_digest(actual, base64.b64decode(digest)) + except (ValueError, TypeError): + return False + + +def verify_password(password, stored): + """-> (ok, needs_rehash). Сравнение внутри libargon2 — за постоянное время.""" + if stored.startswith("$argon2"): + try: + HASHER.verify(stored, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False, False + return True, HASHER.check_needs_rehash(stored) + ok = _verify_scrypt(password, stored) + return ok, ok + + +# Хеш несуществующего пользователя: неверный логин тратит столько же времени (тот же Argon2id), +# сколько неверный пароль, и не выдаёт, существует ли username. +DUMMY_HASH = hash_password(secrets.token_urlsafe(16)) + + +def new_token(): + """Непрозрачный случайный токен: 32 байта = 256 бит энтропии.""" + return secrets.token_urlsafe(32) + + +def token_hash(token): + """В БД лежит только SHA-256 токена (у токена 256 бит энтропии — перебор невозможен, + медленный KDF не нужен): утечка таблицы sessions не даёт готовых сессий.""" + return hashlib.sha256(token.encode("utf-8")).digest() diff --git a/src/selfcheck.py b/src/selfcheck.py new file mode 100644 index 0000000..36eaffc --- /dev/null +++ b/src/selfcheck.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Lab7: самопроверка least privilege ИЗНУТРИ контейнера сервиса (под его собственной identity). + + docker compose exec -T api python /app/src/selfcheck.py + docker compose exec -T worker python /app/src/selfcheck.py + +Ничего секретного не печатает: только HTTP-статусы, имена полей KV, SHA-256-префиксы (8 hex) +значений ключей доступа (чтобы тест сравнил «чей это ключ» без раскрытия), коды ошибок SQL. +Вывод — один JSON в stdout. +""" +import hashlib +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import psycopg # noqa: E402 + +import vault # noqa: E402 + +ROLE = vault.BAO_AUTH_ROLE +OTHER = {"api": "worker", "worker": "api", "relay": "worker", "janitor": "api"}.get(ROLE, "api") + + +def fp(value): + return hashlib.sha256(str(value).encode()).hexdigest()[:8] + + +def bao_checks(): + c = vault.Client() + c.login() + token = c._token + out = {"login": {"policies": c.lease_info["policies"], "ttl": c.lease_info["ttl"], + "renewable": c.lease_info["renewable"]}} + st, data = vault.request("GET", "kv/data/messenger/%s" % ROLE, token=token) + values = data.get("data", {}).get("data", {}) if st == 200 else {} + out["read_own"] = st + out["own_fields"] = sorted(values) + out["own_fingerprints"] = {k: fp(v) for k, v in values.items() if k in ("S3_ACCESS_KEY_ID",)} + probes = { + "read_other_service": ("GET", "kv/data/messenger/%s" % OTHER, None), + "read_migrate_admin": ("GET", "kv/data/messenger/migrate", None), + "list_metadata": ("LIST", "kv/metadata/messenger", None), + "write_own_secret": ("POST", "kv/data/messenger/%s" % ROLE, {"data": {"x": "y"}}), + "write_policy": ("PUT", "sys/policies/acl/evil", {"policy": 'path "*" {capabilities=["sudo"]}'}), + "create_token": ("POST", "auth/token/create", {"policies": ["root"]}), + "seal": ("PUT", "sys/seal", {}), + "list_auth_roles": ("LIST", "auth/cert/certs", None), + } + for name, (method, path, body) in probes.items(): + st, _ = vault.request(method, path, token=token, body=body) + out[name] = st + st, data = vault.request("GET", "auth/token/lookup-self", token=token) + d = data.get("data", {}) + out["lookup_self"] = {"status": st, "policies": d.get("policies"), "ttl": d.get("ttl"), + "explicit_max_ttl": d.get("explicit_max_ttl"), "renewable": d.get("renewable"), + "display_name": d.get("display_name")} + st, data = vault.request("POST", "auth/token/renew-self", token=token, body={}) + out["renew_self"] = st + # Отзыв своего токена: следующий запрос с ним -> 403 (проверка, что отзыв действует сразу). + vault.request("POST", "auth/token/revoke-self", token=token, body={}) + st, _ = vault.request("GET", "kv/data/messenger/%s" % ROLE, token=token) + out["read_after_revoke"] = st + return out, values + + +def db_checks(values): + if "POSTGRES_PASSWORD" not in values: + return None + from psycopg.conninfo import make_conninfo + info = make_conninfo(host=os.environ.get("POSTGRES_HOST", "db"), dbname=os.environ.get("POSTGRES_DB", "messenger"), + user=os.environ["POSTGRES_USER"], password=values["POSTGRES_PASSWORD"], + sslmode="verify-full", sslrootcert=os.environ.get("DB_SSLROOTCERT", "/run/secrets/internal-ca.crt"), + connect_timeout=3) + out = {"user": os.environ["POSTGRES_USER"]} + with psycopg.connect(info, autocommit=True) as conn: + row = conn.execute("SELECT ssl, version, cipher FROM pg_stat_ssl WHERE pid = pg_backend_pid()").fetchone() + out["ssl"] = {"ssl": row[0], "version": row[1], "cipher": row[2]} + r = conn.execute("SELECT rolsuper, rolcreaterole, rolcreatedb, rolbypassrls FROM pg_roles WHERE rolname = current_user").fetchone() + out["role_flags"] = dict(zip(("super", "createrole", "createdb", "bypassrls"), r)) + tests = { + "select_users": "SELECT count(*) FROM users", + "select_sessions": "SELECT count(*) FROM sessions", + "select_messages": "SELECT count(*) FROM messages", + "select_attachments": "SELECT count(*) FROM attachments", + "select_jobs": "SELECT count(*) FROM jobs", + "update_chat_title": "UPDATE chats SET title = title WHERE false", + "delete_messages": "DELETE FROM messages WHERE false", + "truncate_outbox": "TRUNCATE outbox", + "create_table": "CREATE TABLE lab7_probe (x int)", + "drop_users": "DROP TABLE users", + } + for name, q in tests.items(): + try: + with conn.transaction(): + conn.execute(q) + raise _Rollback() + except _Rollback: + out[name] = "allowed" + except psycopg.Error as exc: + out[name] = "denied" if getattr(exc, "sqlstate", "") in ("42501",) else "error:%s" % exc.sqlstate + return out + + +class _Rollback(Exception): + pass + + +def main(): + result = {"role": ROLE, "kv_path": vault.BAO_KV_PATH} + try: + result["openbao"], values = bao_checks() + except Exception as exc: + result["openbao_error"] = type(exc).__name__ + values = {} + try: + result["db"] = db_checks(values) + except Exception as exc: + result["db_error"] = "%s: %s" % (type(exc).__name__, str(exc)[:120]) + env_names = sorted(os.environ) + result["env_has_token"] = [n for n in env_names if n in ("BAO_TOKEN", "VAULT_TOKEN") or n.endswith("_SECRET_ID")] + result["env_secret_like"] = [n for n in env_names if any(x in n for x in ("PASSWORD", "SECRET_ACCESS_KEY")) + and not n.endswith("_FILE")] + result["mounted_secrets"] = sorted(os.listdir("/run/secrets")) if os.path.isdir("/run/secrets") else [] + print(json.dumps(result, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..64c8e24 --- /dev/null +++ b/src/server.py @@ -0,0 +1,1401 @@ +#!/usr/bin/env python3 +"""MTUSI Messenger, lab7: REST API + асинхронная обработка вложений + наблюдаемость + защита. + +Новое в lab7: + * listener API — только mTLS (TLS 1.2+, client cert internal-ca обязателен, SAN DNS клиента + должен быть в MTLS_ALLOWED_CLIENTS, по умолчанию только "lb"); health — отдельный plaintext + listener ТОЛЬКО на 127.0.0.1:8081 контейнера (Docker healthcheck), бизнес-API через него нет; + * пароли — Argon2id (src/security.py), rehash старых хешей при входе; + * секреты (пароль роли БД, ключи S3) — из OpenBao по cert auth (src/vault.py), не из файлов; + * TLS verify-full к PostgreSQL, HTTPS к S3, OTLP/HTTPS с client cert к Collector; + * X-Instance-Id — случайный alias процесса (src/instance.py), не hostname. + +Lab6: + +Новое в lab6 (src/telemetry.py): + * каждый запрос — SERVER span « <шаблон маршрута>» (новый root trace: внешний + traceparent от клиента не принимается — публичная точка входа не доверяет чужому контексту); + в ответе X-Trace-Id (W3C trace-id), X-Request-Id по-прежнему коррелирует ответ с логом; + * метрики http.server.request.duration (histogram, s; _count = счётчик запросов) + {http.request.method, http.route, http.response.status_code, http.response.status_class, + error.type для 5xx} и http.server.active_requests; route — шаблон, не реальный UUID; + * дочерние spans: db <операция> (store.py), s3 <операция> (objstore.py), password hash, + приём тела/проверка изображения upload; jobs.enqueue создаёт PRODUCER span «create», + его traceparent пишется в outbox -> relay -> RabbitMQ -> worker (сквозная трасса); + * структурный лог http.request на каждый запрос (JSON stdout + OTLP -> Loki) c trace_id/span_id. + +Lab5: + * upload: оригинал принимается потоком, проверяется и пишется в staging-prefix S3; + затем ОДНОЙ транзакцией PostgreSQL вложение становится queued, создаются задание + (jobs) и сообщение outbox -> 202 Accepted. API сам в брокер не публикует и resize + не делает: публикует src/relay.py, обрабатывает src/worker.py; + * GET метаданных показывает queued/processing/ready/failed, variants, error_code; + * content: queued/processing/failed -> 409; variant=thumbnail|crop|optimized у ready; + * сообщения можно отправлять с queued/processing/ready вложениями, с failed — 409. + +Из lab4: + * POST /chats/{id}/attachments: сырые bytes PNG/JPEG, потоковый приём во временный + файл без имени (O_TMPFILE: исчезает при закрытии/падении процесса), SHA-256 на лету, + ранние лимиты (размер, сигнатура, пиксели) до чтения всего тела и до декодирования; + * порядок записи: pending-строка в PostgreSQL -> S3 PUT -> ready (+ компенсация, + остальное подбирает janitor после grace period); + * GET /attachments/{id} и /content — только участникам чата, байты через API; + * сообщения ссылаются на ready-вложения своего чата (attachment_ids). + +Из lab3 (MTUSI Messenger, lab3: REST API для запуска несколькими репликами за балансировщиком): + +Из lab2: PostgreSQL как источник истины, сессии (Bearer) в PostgreSQL, scrypt. +Новое в lab3: + * каждый ответ содержит X-Instance-Id (стабилен на жизнь процесса); + * POST сообщения требует Idempotency-Key, идемпотентность общая через PostgreSQL; + * миграции НЕ применяются репликами: их однократно выполняет src/migrate.py; + * SIGTERM: readiness -> 503, пауза на исключение из LB, дообслуживание и выход; + * пароль БД читается из файла секрета (POSTGRES_PASSWORD_FILE / DATABASE_URL_FILE); + * не более PASSWORD_HASH_CONCURRENCY одновременных scrypt (защита от OOM при шторме входов); + * /health/ready смотрит на фоновую проверку БД со своим соединением, а не на пул запросов. +""" +import base64 +import binascii +import contextlib +import hashlib +import json +import logging +import os +import re +import signal +import socket +import ssl +import sys +import tempfile +import threading +import unicodedata +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import psycopg +from opentelemetry import trace +from psycopg.conninfo import make_conninfo +from psycopg_pool import PoolTimeout + +import images +import jobs +import objstore +import security +import telemetry +import tlsutil +import vault +from instance import INSTANCE_ID +from telemetry import SpanKind, Status, StatusCode +from objstore import ObjectMissing, StorageUnavailable +from store import AttachmentFailed, AttachmentNotFound, Conflict, Store, UnknownMember, from_micros + +HOST = os.environ.get("APP_HOST", "0.0.0.0") +PORT = int(os.environ.get("APP_PORT", "8443")) +APP_ENV = os.environ.get("APP_ENV", "lab") +SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) +MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", str(Path(__file__).resolve().parents[1] / "migrations")) +DB_POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10")) +DB_STATEMENT_TIMEOUT_MS = int(os.environ.get("DB_STATEMENT_TIMEOUT_MS", "5000")) +# Graceful shutdown: сколько секунд после SIGTERM продолжаем обслуживать запросы с ready=503, +# чтобы LB успел исключить реплику (>= inter * fall проверки HAProxy), и сколько ждём in-flight. +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "3")) +SHUTDOWN_TIMEOUT_SECONDS = float(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "10")) +# Таймаут сокета клиента: медленный/зависший клиент не держит поток бесконечно. +CLIENT_SOCKET_TIMEOUT_SECONDS = float(os.environ.get("CLIENT_SOCKET_TIMEOUT_SECONDS", "30")) +# Lab7: alias реплики — случайный (src/instance.py), не hostname/IP; stop-one.sh находит +# контейнер по файлу /tmp/instance-id внутри него. +# Версия образа (build-arg APP_VERSION): видна в X-App-Version, нужна для проверки rolling update. +APP_VERSION = os.environ.get("APP_VERSION", "dev") +MAX_BODY_BYTES = 1024 * 1024 +# scrypt берёт ~32 MiB на проверку пароля (128 * r * N), лимит реплики 256 MiB. +# Без ограничения ~7 одновременных входов дают OOM-kill (проверено: evidence/before-fix). +# Больше 2 параллельных хешей на 0.5 CPU пропускную способность всё равно не увеличивает. +PASSWORD_HASH_CONCURRENCY = int(os.environ.get("PASSWORD_HASH_CONCURRENCY", "2")) +# Сколько запрос ждёт свободный слот scrypt, прежде чем получить 503 + Retry-After. +# Коротко: ожидающие входы не должны занимать слоты LB (maxconn) у обычных запросов. +PASSWORD_HASH_WAIT_SECONDS = float(os.environ.get("PASSWORD_HASH_WAIT_SECONDS", "0.5")) +# Readiness по БД: фоновый поток со СВОИМ соединением (не из пула запросов) пингует БД +# раз в DB_HEALTH_INTERVAL_SECONDS; реплика «готова», если последний успех не старше +# READY_STALE_SECONDS. Так проверка готовности не конкурирует с трафиком за пул. +DB_HEALTH_INTERVAL_SECONDS = float(os.environ.get("DB_HEALTH_INTERVAL_SECONDS", "1")) +READY_STALE_SECONDS = float(os.environ.get("READY_STALE_SECONDS", "3")) + +# --- Lab4: вложения --- +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) # 10 MiB +MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +# Весь upload (приём тела) должен уложиться в этот срок, иначе 400 и разрыв соединения. +# Он намного меньше grace period janitor (ATTACHMENT_GRACE_SECONDS, 1 ч). +UPLOAD_TIMEOUT_SECONDS = float(os.environ.get("UPLOAD_TIMEOUT_SECONDS", "60")) +# Одновременных загрузок на реплику и сколько ждать слот до 503. +UPLOAD_CONCURRENCY = int(os.environ.get("UPLOAD_CONCURRENCY", "8")) +UPLOAD_SLOT_WAIT_SECONDS = float(os.environ.get("UPLOAD_SLOT_WAIT_SECONDS", "5")) +# Пробное декодирование JPEG (draft 1/8) — единственный шаг, где память зависит +# от размера кадра; ограничиваем параллелизм отдельно. +DECODE_CONCURRENCY = int(os.environ.get("DECODE_CONCURRENCY", "2")) +# S3 PUT — самый «дорогой» по RAM шаг: minio-py держит в памяти part 5 MiB (с копиями +# для подписи), ~20 MiB на загрузку. Ограничиваем параллельные PUT на реплику, остальные +# ждут в очереди (тело уже принято на диск). Замер: evidence/upload-memory-*.json. +S3_PUT_CONCURRENCY = int(os.environ.get("S3_PUT_CONCURRENCY", "3")) +S3_PUT_WAIT_SECONDS = float(os.environ.get("S3_PUT_WAIT_SECONDS", "30")) +UPLOAD_TMP_DIR = os.environ.get("UPLOAD_TMP_DIR") or None +if UPLOAD_TMP_DIR: # lab7: /tmp — tmpfs поверх read-only rootfs, каталог создаётся при старте + os.makedirs(UPLOAD_TMP_DIR, mode=0o700, exist_ok=True) +S3_HEALTH_INTERVAL_SECONDS = float(os.environ.get("S3_HEALTH_INTERVAL_SECONDS", "1")) +S3_READY_STALE_SECONDS = float(os.environ.get("S3_READY_STALE_SECONDS", "3")) +# Сразу после старта реплики первая проверка S3 ещё не завершилась: upload ждёт её до +# этого срока, а не получает мгновенный 503. При подтверждённом отказе S3 не ждём. +S3_STARTUP_WAIT_SECONDS = float(os.environ.get("S3_STARTUP_WAIT_SECONDS", "5")) +# Только для учебного эксперимента отказов (student-tests/consistency_faults.py): +# при FAULT_INJECTION=1 заголовок X-Fault-Inject роняет процесс/запись в заданной точке. +FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1" +# Lab6: точка slow-handler (X-Fault-Inject: slow-handler) — «медленный код обработчика» без БД, +# для сравнения в трассе с медленной БД. Только при FAULT_INJECTION=1. +FAULT_SLOW_HANDLER_SECONDS = float(os.environ.get("FAULT_SLOW_HANDLER_SECONDS", "1.5")) +# Lab5: версия pipeline новых заданий и лимит попыток (подробности — src/jobs.py, worker.py). +PIPELINE_VERSION = int(os.environ.get("PIPELINE_VERSION", "1")) +JOB_MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +ALLOWED_IMAGE_TYPES = ("image/png", "image/jpeg") +KNOWN_VARIANTS = ("thumbnail", "crop", "optimized") +UPLOAD_READ_CHUNK = 64 * 1024 +# Ранний отказ upload (401/404/413/415): остаток тела до этого объёма дочитывается и +# выбрасывается, чтобы клиент, ещё передающий файл, получил ответ, а не обрыв соединения. +# Больше — соединение просто закрывается (не тратим трафик на заведомо огромное тело). +REJECT_DRAIN_BYTES = int(os.environ.get("REJECT_DRAIN_BYTES", str(2 * 10485760))) +# Общий срок приёма JSON-тела (до 1 MiB), аудит F03. +JSON_BODY_TIMEOUT_SECONDS = float(os.environ.get("JSON_BODY_TIMEOUT_SECONDS", "10")) +# Bidi-override и изолирующие символы позволяют подделать видимое расширение (gpj.exe). +_BIDI_CONTROLS = set("\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069\u200e\u200f") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +DEFAULT_LIMIT = 50 +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + +log = logging.getLogger("messenger") + +# --- Lab6: метрики HTTP (низкокардинальные атрибуты) --- +HTTP_DURATION = telemetry.meter.create_histogram( + "http.server.request.duration", unit="s", + description="Duration of HTTP server requests (count = number of requests)") +HTTP_ACTIVE = telemetry.meter.create_up_down_counter( + "http.server.active_requests", unit="{request}", description="In-flight HTTP requests") +_KNOWN_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} +# Проверки HAProxy (/health/ready каждую секунду на реплику) в метриках есть, но trace и +# INFO-лог на каждую не создаются: это шум (2 trace/с) без диагностической ценности. +# Неуспешная проверка (не 200) логируется всегда. OTEL_TRACE_HEALTH=1 — трассировать и их. +TRACE_HEALTH = os.environ.get("OTEL_TRACE_HEALTH", "0") == "1" + + +def _secret(name): + """Lab7: секреты сервисов — из OpenBao (src/vault.py); NAME_FILE/NAME — только для job/тестов.""" + return vault.secret(name) + + +def database_conninfo(): + """DATABASE_URL(_FILE) либо набор POSTGRES_*; пароль — только из секрета/окружения.""" + options = "-c statement_timeout=%d" % DB_STATEMENT_TIMEOUT_MS + url = vault.secret("DATABASE_URL", required=False) + if url: + return make_conninfo(url, connect_timeout=3, options=options, sslmode="verify-full", + sslrootcert=os.environ.get("DB_SSLROOTCERT", tlsutil.CA_FILE)) + return make_conninfo( + host=os.environ.get("POSTGRES_HOST", "localhost"), + port=os.environ.get("POSTGRES_PORT", "5432"), + dbname=os.environ.get("POSTGRES_DB", "messenger"), + user=os.environ.get("POSTGRES_USER", "messenger"), + password=_secret("POSTGRES_PASSWORD"), + connect_timeout=3, + # Lab7: TLS к PostgreSQL с проверкой CA и hostname (verify-full); pg_hba отвергает plaintext. + sslmode=os.environ.get("DB_SSLMODE", "verify-full"), + sslrootcert=os.environ.get("DB_SSLROOTCERT", tlsutil.CA_FILE), + options=options, + ) + + +class DbHealth: + """Фоновая проверка PostgreSQL для /health/ready. + + Отдельное соединение вне пула: при исчерпанном пуле (перегрузка) readiness не ждёт + PoolTimeout и не роняет все реплики в DOWN одновременно. Зависание БД/сети тоже + видно: если успешного пинга нет дольше stale_after, реплика не готова.""" + + def __init__(self, conninfo, interval, stale_after): + self.conninfo = conninfo + self.interval = interval + self.stale_after = stale_after + self.last_ok = None # time.monotonic() последнего успешного SELECT 1 + self._stop = threading.Event() + self._thread = None + + def start(self): + self._thread = threading.Thread(target=self._loop, name="db-health", daemon=True) + self._thread.start() + + def stop(self): + self._stop.set() + + def ok(self): + last = self.last_ok + return last is not None and time.monotonic() - last <= self.stale_after + + def _loop(self): + conn, healthy = None, None + while not self._stop.is_set(): + try: + if conn is None or conn.closed: + conn = psycopg.connect(self.conninfo, autocommit=True) + conn.execute("SELECT 1") + self.last_ok = time.monotonic() + if healthy is not True: + log.info("db health: ok", extra={"event": "db.health.ok"}) + healthy = True + except Exception as exc: + if healthy is not False: + log.warning("db health: failing (%s)", type(exc).__name__, + extra={"event": "db.health.failing", "error.type": type(exc).__name__}) + healthy = False + if conn is not None: + try: + conn.close() + except Exception: + pass + conn = None + self._stop.wait(self.interval) + if conn is not None: + conn.close() + + +DB_CONNINFO = database_conninfo() +STORE = Store(DB_CONNINFO, pool_size=DB_POOL_SIZE) +DB_HEALTH = DbHealth(DB_CONNINFO, DB_HEALTH_INTERVAL_SECONDS, READY_STALE_SECONDS) +OBJ = objstore.ObjectStore(objstore.make_client(), S3_BUCKET) +S3_HEALTH = objstore.S3Health(OBJ, S3_HEALTH_INTERVAL_SECONDS, S3_READY_STALE_SECONDS, INSTANCE_ID) +_UPLOAD_SLOTS = threading.BoundedSemaphore(UPLOAD_CONCURRENCY) +_DECODE_SLOTS = threading.BoundedSemaphore(DECODE_CONCURRENCY) +_PUT_SLOTS = threading.BoundedSemaphore(S3_PUT_CONCURRENCY) +DRAINING = threading.Event() # выставляется по SIGTERM +_INFLIGHT = 0 +_INFLIGHT_LOCK = threading.Lock() +_HASH_SLOTS = threading.BoundedSemaphore(PASSWORD_HASH_CONCURRENCY) + + +# -------------------------------------------------------------------------- +# Ошибки API +# -------------------------------------------------------------------------- +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message="Неверный запрос"): + return ApiError(400, "invalid_request", message) + + +def unauthorized(challenge=BEARER_CHALLENGE): + # Одинаковый ответ для любых причин: не раскрываем, что именно не так. + return ApiError(401, "unauthorized", "Требуется действительная аутентификация", + {"WWW-Authenticate": challenge}) + + +def not_found(): + return ApiError(404, "not_found", "Объект не найден") + + +def storage_unavailable(): + return ApiError(503, "unavailable", "Хранилище файлов временно недоступно", {"Retry-After": "1"}) + + +def _api_error_type(err): + """error.type для 5xx-ответов приложения: перегрузка (слоты) отдельно от недоступности.""" + text = err.message or "" + if "Хранилище" in text: + return "storage_unavailable" + if "перегруж" in text or "Слишком много" in text: + return "overloaded" + if "останавливается" in text: + return "draining" + return "unavailable" + + +class ClientGone(Exception): + """Клиент оборвал передачу тела: ответить уже некому.""" + + +STREAMED = object() # обработчик сам отправил ответ (download) + + +@contextlib.contextmanager +def bounded_slot(sem, wait, message): + if not sem.acquire(timeout=wait): + raise ApiError(503, "unavailable", message, {"Retry-After": "1"}) + try: + yield + finally: + sem.release() + + +@contextlib.contextmanager +def password_hash_slot(): + """Ограничивает число одновременных scrypt (память!). Нет слота за + PASSWORD_HASH_WAIT_SECONDS — честный 503 с Retry-After, а не OOM всей реплики.""" + if not _HASH_SLOTS.acquire(timeout=PASSWORD_HASH_WAIT_SECONDS): + raise ApiError(503, "unavailable", "Сервис перегружен, повторите запрос позже", + {"Retry-After": "1"}) + try: + yield + finally: + _HASH_SLOTS.release() + + +# -------------------------------------------------------------------------- +# Валидация +# -------------------------------------------------------------------------- +def parse_uuid(value): + try: + return str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError): + return None + + +def require_object(body, allowed, required): + if not isinstance(body, dict): + raise bad_request("Тело запроса должно быть JSON-объектом") + if set(body) - set(allowed): + raise bad_request("Лишние поля в запросе") + if any(k not in body for k in required): + raise bad_request("Не хватает обязательных полей") + + +def require_text(value, field, max_len, min_len=1): + if not isinstance(value, str): + raise bad_request("Поле %s должно быть строкой" % field) + if not value.strip(): + raise bad_request("Поле %s не должно быть пустым" % field) + if "\x00" in value: + raise bad_request("Поле %s содержит недопустимый символ" % field) + if not min_len <= len(value) <= max_len: # len() считает Unicode code points + raise bad_request("Поле %s имеет недопустимую длину" % field) + return value + + +def encode_cursor(kind, scope, key): + raw = json.dumps({"k": kind, "s": scope, "t": key[0], "i": key[1]}, separators=(",", ":")) + return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + + +def decode_cursor(cursor, kind, scope): + try: + padded = cursor + "=" * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(padded.encode("ascii"))) + if data["k"] != kind or data["s"] != scope: + raise ValueError + t, i = data["t"], parse_uuid(data["i"]) + if not isinstance(t, int) or isinstance(t, bool) or i is None or abs(t) > 10 ** 17: + raise ValueError + from_micros(t) # Проверяем границы datetime до передачи курсора в Store. + return (t, i) + except (ValueError, KeyError, TypeError, UnicodeError, binascii.Error, OverflowError): + raise bad_request("Неверный cursor") + + +def page_params(query): + def single(name): + values = query.get(name) + if values is None: + return None + if len(values) != 1: + raise bad_request("Параметр %s указан несколько раз" % name) + return values[0] + + limit_raw = single("limit") + if limit_raw is None: + limit = DEFAULT_LIMIT + else: + if not re.fullmatch(r"[0-9]{1,3}", limit_raw) or not 1 <= int(limit_raw) <= 100: + raise bad_request("limit должен быть целым от 1 до 100") + limit = int(limit_raw) + cursor = single("cursor") + if cursor == "": + raise bad_request("Неверный cursor") + return limit, cursor + + +# -------------------------------------------------------------------------- +# Обработчики +# -------------------------------------------------------------------------- +def h_live(req): + return 200, {"status": "ok"} + + +def h_ready(req): + # После SIGTERM реплика сообщает LB, что новые запросы ей слать не нужно. + if DRAINING.is_set(): + raise ApiError(503, "unavailable", "Реплика останавливается") + # Критичная зависимость одна — PostgreSQL. Без неё обслуживать запросы нельзя. + # Состояние берётся из фоновой проверки (DbHealth), а не из пула запросов. + if not DB_HEALTH.ok(): + raise ApiError(503, "unavailable", "Сервис временно недоступен") + return 200, {"status": "ok"} + + +def h_create_user(req): + body = req.json_body() + fields = ("username", "display_name", "password") + require_object(body, fields, fields) + username = body["username"] + if not isinstance(username, str) or not USERNAME_RE.fullmatch(username): + raise bad_request("username должен соответствовать [a-z0-9_]{3,32}") + display_name = require_text(body["display_name"], "display_name", 100) + password = require_text(body["password"], "password", 128, min_len=12) + with password_hash_slot(), telemetry.child_span("security.password_hash"): + password_hash = security.hash_password(password) + try: + return 201, STORE.create_user(username, display_name, password_hash) + except Conflict: + raise ApiError(409, "conflict", "Имя пользователя уже занято") + + +def h_create_session(req): + """Вход: только Authorization: Basic, без JSON body.""" + header = req.headers.get("Authorization", "") + scheme, _, encoded = header.partition(" ") + failure = unauthorized(BASIC_CHALLENGE) + if scheme.lower() != "basic": + raise failure + try: + decoded = base64.b64decode(encoded.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeError): + raise failure + username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо + if not sep: + raise failure + user, stored_hash = (None, None) + if USERNAME_RE.fullmatch(username): + user, stored_hash = STORE.get_credentials(username) + # Проверяем scrypt всегда (для неизвестного логина — с фиктивным хешем): + # время ответа не выдаёт, существует ли пользователь. + with password_hash_slot(), telemetry.child_span("security.password_verify"): + ok, needs_rehash = security.verify_password(password, stored_hash or security.DUMMY_HASH) + if user is None or not ok: + raise failure + if needs_rehash: + # Lab7: старая схема/параметры -> Argon2id с текущими параметрами (plaintext есть сейчас). + with password_hash_slot(), telemetry.child_span("security.password_rehash"): + STORE.update_password_hash(user["id"], stored_hash, security.hash_password(password)) + log.info("password rehashed to current argon2id params", extra={"event": "security.password_rehash"}) + token = security.new_token() + expires_at = STORE.create_session(user["id"], security.token_hash(token), SESSION_TTL_SECONDS) + return 201, {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +def h_delete_session(req): + req.current_user() + STORE.revoke_session(req.token_hash) + return 204, None + + +def h_me(req): + return 200, req.current_user() + + +def h_create_chat(req): + user = req.current_user() + body = req.json_body() + require_object(body, ("title", "member_ids"), ("title", "member_ids")) + title = require_text(body["title"], "title", 100) + members = body["member_ids"] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise bad_request("member_ids должен быть списком из 1–100 UUID") + normalized = [] + for m in members: + mid = parse_uuid(m) if isinstance(m, str) else None + if mid is None: + raise bad_request("member_ids должен содержать UUID-строки") + normalized.append(mid) + if len(set(normalized)) != len(normalized): + raise bad_request("Дубликаты в member_ids") + if user["id"] not in normalized: + raise bad_request("Создатель должен входить в member_ids") + try: + return 201, STORE.create_chat(title, normalized) + except UnknownMember: + raise bad_request("Неизвестный участник чата") + + +def h_list_chats(req): + user = req.current_user() + limit, cursor = page_params(req.query) + after = decode_cursor(cursor, "chats", user["id"]) if cursor else None + items, last = STORE.list_chats(user["id"], after, limit) + return 200, {"items": items, "next_cursor": encode_cursor("chats", user["id"], last) if last else None} + + +def _chat_or_404(user, chat_id): + chat = STORE.get_chat_for(user["id"], chat_id) + if chat is None: + raise not_found() + return chat + + +def h_get_chat(req, chat_id): + user = req.current_user() + return 200, _chat_or_404(user, req.path_uuid(chat_id)) + + +def request_fingerprint(text, attachment_ids=()): + """SHA-256 канонического представления тела: порядок ключей JSON не важен. + attachment_ids (с lab4) по умолчанию [] — отсутствие поля и [] семантически равны; + ключи lab3 посчитаны с [] и остаются валидными.""" + canonical = json.dumps({"attachment_ids": list(attachment_ids), "text": text}, sort_keys=True, + separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +def h_create_message(req, chat_id): + # Порядок: сессия -> доступ к чату -> тело -> ключ. Повтор с ключом снова проверяет + # действительность сессии и членство, поэтому отозванный токен не получит старый ответ. + user = req.current_user() + cid = req.path_uuid(chat_id) + _chat_or_404(user, cid) # посторонний получает 404 + body = req.json_body() + require_object(body, ("text", "attachment_ids"), ("text",)) + attachment_ids = body.get("attachment_ids", []) + if not isinstance(attachment_ids, list) or len(attachment_ids) > 10: + raise bad_request("attachment_ids должен быть списком из 0–10 UUID") + normalized = [] + for a in attachment_ids: + aid = parse_uuid(a) if isinstance(a, str) else None + if aid is None: + raise bad_request("attachment_ids должен содержать UUID-строки") + normalized.append(aid) + if len(set(normalized)) != len(normalized): + raise bad_request("Дубликаты в attachment_ids") + text = body["text"] + if not isinstance(text, str) or "\x00" in text or len(text) > 4000: + raise bad_request("Поле text должно быть строкой до 4000 символов") + if not normalized: + text = require_text(text, "text", 4000) # без вложений пустой/пробельный текст нельзя + key = req.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise bad_request("Требуется Idempotency-Key: [A-Za-z0-9._:-]{1,128}") + try: + result = STORE.add_message(user["id"], cid, text, normalized, key, + request_fingerprint(text, normalized), IDEMPOTENCY_TTL_SECONDS) + except Conflict: + raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим сообщением") + except AttachmentNotFound: + raise not_found() + except AttachmentFailed: + raise ApiError(409, "conflict", "Вложение не обработано (failed) и не может быть прикреплено") + if result is None: + raise not_found() + message, replayed = result + if replayed: + req.extra_headers["Idempotency-Replayed"] = "true" + return 201, message + + +def h_list_messages(req, chat_id): + user = req.current_user() + cid = req.path_uuid(chat_id) + _chat_or_404(user, cid) + limit, cursor = page_params(req.query) + fault(req, "slow-handler") + after = decode_cursor(cursor, "messages", cid) if cursor else None + items, last = STORE.list_messages(cid, after, limit) + return 200, {"items": items, "next_cursor": encode_cursor("messages", cid, last) if last else None} + + +# -------------------------------------------------------------------------- +# Вложения (lab4) +# -------------------------------------------------------------------------- +def single_query(req, name): + values = req.query.get(name) + if values is None: + return None + if len(values) != 1: + raise bad_request("Параметр %s указан несколько раз" % name) + return values[0] + + +def validate_filename(name): + """Отображаемое имя: не путь и не ключ S3. Опасное имя -> 400.""" + if name is None: + raise bad_request("Требуется параметр filename") + if not 1 <= len(name) <= 255 or not name.strip(): + raise bad_request("filename должен содержать 1–255 символов") + if name in (".", ".."): + raise bad_request("Недопустимый filename") + for ch in name: + if ch in "/\\" or ch in _BIDI_CONTROLS or unicodedata.category(ch) in ("Cc", "Cs"): + raise bad_request("filename содержит недопустимые символы") + return name + + +def fault(req, point): + """Учебная инъекция отказа; выключена, если FAULT_INJECTION != 1.""" + if not FAULT_INJECTION or req.headers.get("X-Fault-Inject") != point: + return + log.warning("FAULT INJECTION %s request_id=%s", point, req.request_id, + extra={"event": "fault.injected", "fault.point": point, "request_id": req.request_id}) + if point == "slow-handler": + with telemetry.child_span("handler.compute", attributes={"fault.point": point}): + time.sleep(FAULT_SLOW_HANDLER_SECONDS) + return + if point.startswith("crash-"): + os._exit(86) # как SIGKILL: без finally, компенсаций и ответа + raise psycopg.OperationalError("injected fault") + + +def receive_body(req, tmp, length, declared_type): + """Читает ровно length байт кусками по 64 KiB во временный файл, считая SHA-256. + + Память на загрузку — один буфер; общий срок UPLOAD_TIMEOUT_SECONDS; уже по + первым 33 байтам проверяются сигнатура (подмена MIME) и пиксели PNG из IHDR.""" + digest = hashlib.sha256() + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + received = 0 + head = b"" + checked_head = False + while received < length: + left = deadline - time.monotonic() + if left <= 0: + raise bad_request("Загрузка не завершена вовремя") + req.connection.settimeout(min(left, CLIENT_SOCKET_TIMEOUT_SECONDS)) + try: + # read1, а не read: read(n) ждёт ВСЕ n байт за сколько угодно recv, и клиент, + # присылающий по байту, обходил общий срок (аудит F03). read1 — не больше одного + # recv, поэтому срок проверяется перед каждым фактическим чтением сокета. + chunk = req.rfile.read1(min(UPLOAD_READ_CHUNK, length - received)) + except (TimeoutError, socket.timeout): + raise bad_request("Загрузка не завершена вовремя") + except (ConnectionError, OSError): + raise ClientGone() + if not chunk: + raise ClientGone() # EOF раньше Content-Length: передача прервана + received += len(chunk) + req.body_left = length - received + digest.update(chunk) + tmp.write(chunk) + if not checked_head and (len(head) < 33): + head += chunk[:33 - len(head)] + if not checked_head and (len(head) >= 33 or received == length): + checked_head = True + if images.sniff(head) != declared_type: + raise ApiError(415, "unsupported_media_type", + "Содержимое не соответствует заявленному типу изображения") + dims = images.png_header_pixels(head) if declared_type == "image/png" else None + if dims and dims[0] * dims[1] > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Изображение больше 40 млн пикселей") + req.body_consumed = True + req.connection.settimeout(CLIENT_SOCKET_TIMEOUT_SECONDS) + tmp.flush() + return digest.hexdigest() + + +def h_upload_attachment(req, chat_id): + # Порядок: аутентификация -> доступ к чату -> параметры/заголовки -> только потом тело. + user = req.current_user() + cid = req.path_uuid(chat_id) + _chat_or_404(user, cid) + filename = validate_filename(single_query(req, "filename")) + ctype = (req.headers.get("Content-Type") or "").split(";")[0].strip().lower() + if ctype not in ALLOWED_IMAGE_TYPES: + raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg") + if req.headers.get("Transfer-Encoding"): + raise bad_request("Требуется Content-Length (chunked не поддерживается)") + raw_length = req.headers.get("Content-Length") + if raw_length is None or not raw_length.isdigit(): + raise bad_request("Требуется Content-Length") + length = int(raw_length) + if length == 0: + raise ApiError(415, "unsupported_media_type", "Пустой файл не является изображением") + if length > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + if not S3_HEALTH.ok() and not S3_HEALTH.wait_startup(S3_STARTUP_WAIT_SECONDS): + raise storage_unavailable() # не принимаем 10 MiB, которые некуда сохранить + with bounded_slot(_UPLOAD_SLOTS, UPLOAD_SLOT_WAIT_SECONDS, "Слишком много загрузок, повторите позже"): + with tempfile.TemporaryFile(dir=UPLOAD_TMP_DIR) as tmp: + with telemetry.child_span("http.receive_body", attributes={"http.request.body.size": length, + "messenger.content_type": ctype}): + sha256 = receive_body(req, tmp, length, ctype) + try: + with bounded_slot(_DECODE_SLOTS, UPLOAD_SLOT_WAIT_SECONDS, "Сервис перегружен, повторите позже"), \ + telemetry.child_span("image.validate", attributes={"messenger.content_type": ctype}): + width, height = images.validate(tmp, length, ctype, MAX_IMAGE_PIXELS) + except images.TooManyPixels: + raise ApiError(413, "payload_too_large", "Изображение больше 40 млн пикселей") + except images.ResourceLimit as exc: + log.warning("image check exceeded limits (%s) request_id=%s", exc, req.request_id) + raise ApiError(413, "payload_too_large", "Изображение требует слишком много ресурсов для проверки") + except images.InvalidImage as exc: + log.info("rejected image (%s) request_id=%s", exc, req.request_id) + raise ApiError(415, "unsupported_media_type", "Файл повреждён или не является PNG/JPEG") + att_id = str(uuid.uuid4()) + # Технический ID синтетических данных в трассе: по нему upload связывается с job. + trace.get_current_span().set_attribute("messenger.attachment.id", att_id) + # Ключи назначает сервер. Конечный ключ оригинала детерминирован ключом эффекта + # (attachment_id, pipeline_version); staging — случайный, filename не участвует. + object_key = "attachments/%s/v%d/original" % (att_id, PIPELINE_VERSION) + staging_key = "staging/" + uuid.uuid4().hex + # 1) намерение (pending, скрыто) фиксируется ДО записи в staging + STORE.create_pending_attachment(att_id, cid, user["id"], filename, ctype, length, + sha256, width, height, object_key, staging_key, PIPELINE_VERSION) + fault(req, "crash-after-pending") + # 2) полный проверенный оригинал -> устойчивый staging (приватный prefix S3) + tmp.seek(0) + try: + with bounded_slot(_PUT_SLOTS, S3_PUT_WAIT_SECONDS, "Хранилище перегружено, повторите позже"): + OBJ.put(staging_key, tmp, length, ctype) + except ApiError: + _forget_pending(att_id) # PUT не начинался: объекта нет + raise + except StorageUnavailable: + # Неудачный PUT неоднозначен: сначала удаляем объект, потом строку; + # если S3 не ответил, pending-строку и объект уберёт janitor после grace. + _compensate_object(staging_key, att_id) + raise storage_unavailable() + fault(req, "crash-after-staging") + # 3) одна транзакция: queued + jobs + outbox. После COMMIT работа принята. + # При ошибке НЕ удаляем staging: COMMIT мог пройти (обрыв соединения на COMMIT), + # а удалить оригинал принятой работы нельзя. Pending-остаток уберёт janitor. + attachment = STORE.enqueue_attachment(att_id, PIPELINE_VERSION, JOB_MAX_ATTEMPTS) + if attachment is None: + raise storage_unavailable() + fault(req, "crash-after-commit") # 202 не ушёл, но задание уже будет выполнено + return 202, attachment + + +def _forget_pending(att_id): + try: + STORE.delete_pending_attachment(att_id) + except Exception as exc: + log.warning("pending attachment left for janitor (%s)", type(exc).__name__) + + +def _compensate_object(object_key, att_id): + """PUT в staging не удался/неоднозначен: удаляем объект, затем pending-строку. + Если не вышло, pending-строка остаётся, и janitor удалит объект после grace period.""" + try: + OBJ.delete(object_key) + except Exception as exc: + log.warning("compensation failed, object left for janitor (%s)", type(exc).__name__) + return + _forget_pending(att_id) + + +def _attachment_or_404(req, attachment_id): + user = req.current_user() + found = STORE.get_attachment_for(user["id"], req.path_uuid(attachment_id)) + if found is None: + raise not_found() # чужое и несуществующее неразличимы + return found + + +def h_get_attachment(req, attachment_id): + attachment, _key = _attachment_or_404(req, attachment_id) + return 200, attachment + + +def h_download_attachment(req, attachment_id): + attachment, object_key = _attachment_or_404(req, attachment_id) + variant = single_query(req, "variant") + if variant not in (None, "original") and variant not in KNOWN_VARIANTS: + raise bad_request("Неизвестный variant") + if attachment["status"] in ("queued", "processing"): + raise ApiError(409, "conflict", "Вложение ещё обрабатывается", {"Retry-After": "1"}) + if attachment["status"] == "failed": + raise ApiError(409, "conflict", "Обработка вложения завершилась ошибкой") + if variant in KNOWN_VARIANTS: + meta = next((v for v in attachment["variants"] if v["name"] == variant), None) + object_key = STORE.get_variant_key(attachment["id"], variant) if meta else None + if object_key is None: + raise not_found() # у ready-объекта такого варианта нет + content_type, size, sha, filename = meta["content_type"], meta["size_bytes"], meta["sha256"], None + else: + content_type, size, sha = attachment["content_type"], attachment["size_bytes"], attachment["sha256"] + filename = attachment["filename"] + if S3_HEALTH.known_down(): + raise storage_unavailable() # подтверждённый отказ S3: сразу 503, без ожидания таймаута + try: + resp = OBJ.open(object_key) + except ObjectMissing: + log.error("object missing for ready attachment %s request_id=%s", attachment["id"], req.request_id) + raise storage_unavailable() + except StorageUnavailable: + raise storage_unavailable() + # Аудит F09: сначала весь объект (<= 10 MiB) во временный файл со сверкой размера и + # SHA-256 с метаданными, и только потом 200. Раньше заголовки 200 уходили до проверки, + # и повреждённый объект той же длины клиент получал как успешный ответ. + with tempfile.TemporaryFile(dir=UPLOAD_TMP_DIR) as tmp: + digest = hashlib.sha256() + got = 0 + try: + for chunk in resp.stream(objstore.STREAM_CHUNK): + got += len(chunk) + if got > size: + break + digest.update(chunk) + tmp.write(chunk) + except Exception as exc: + log.warning("download from storage interrupted (%s) request_id=%s", type(exc).__name__, req.request_id) + raise storage_unavailable() + finally: + resp.close() + resp.release_conn() + if got != size or digest.hexdigest() != sha: + log.error("stored object differs from metadata (attachment %s, variant %s) request_id=%s", + attachment["id"], variant or "original", req.request_id) + raise storage_unavailable() + headers = { + "Content-Type": content_type, + "Content-Length": str(size), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"%s"' % sha, + } + if filename: + headers["Content-Disposition"] = "inline; filename*=UTF-8''" + _rfc5987(filename) + tmp.seek(0) + req.send_raw_headers(200, headers) + try: + for chunk in iter(lambda: tmp.read(objstore.STREAM_CHUNK), b""): + req.wfile.write(chunk) + except OSError as exc: + log.info("client went away during download (%s) request_id=%s", type(exc).__name__, req.request_id) + req.close_connection = True + return 200, STREAMED + + +def _rfc5987(value): + from urllib.parse import quote + return quote(value, safe="") + + +ROUTES = [ + # (метод, шаблон пути, обработчик, http.route — шаблон для метрик/spans, не реальный UUID) + ("GET", re.compile(r"^/health/live$"), h_live, "/health/live"), + ("GET", re.compile(r"^/health/ready$"), h_ready, "/health/ready"), + ("POST", re.compile(r"^/api/v1/users$"), h_create_user, "/api/v1/users"), + ("GET", re.compile(r"^/api/v1/users/me$"), h_me, "/api/v1/users/me"), + ("POST", re.compile(r"^/api/v1/auth/sessions$"), h_create_session, "/api/v1/auth/sessions"), + ("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), h_delete_session, "/api/v1/auth/sessions/current"), + ("POST", re.compile(r"^/api/v1/chats$"), h_create_chat, "/api/v1/chats"), + ("GET", re.compile(r"^/api/v1/chats$"), h_list_chats, "/api/v1/chats"), + ("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), h_get_chat, "/api/v1/chats/{chat_id}"), + ("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), h_create_message, "/api/v1/chats/{chat_id}/messages"), + ("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), h_list_messages, "/api/v1/chats/{chat_id}/messages"), + ("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), h_upload_attachment, + "/api/v1/chats/{chat_id}/attachments"), + ("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), h_get_attachment, "/api/v1/attachments/{attachment_id}"), + ("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), h_download_attachment, + "/api/v1/attachments/{attachment_id}/content"), +] +# Неизвестный путь — одно значение route (raw URL в атрибуты метрик не попадает). +UNMATCHED_ROUTE = "unmatched" + + +def match_route(method, path): + """(обработчик, groups, route) либо (None, (), route) — route шаблона пути или unmatched.""" + route = UNMATCHED_ROUTE + for m, pattern, func, template in ROUTES: + match = pattern.match(path) + if match: + if m == method: + return func, match.groups(), template + route = template # путь известен, метод нет (ответ 404, как в lab5) + return None, (), route + + +# Обработчики, которые сами читают тело запроса потоком (не через MAX_BODY_BYTES в памяти). +STREAMING_BODY = {h_upload_attachment} + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + # Lab7: TCP_NODELAY. Заголовки и тело ответа уходят отдельными TLS-записями; с Nagle вторая + # запись ждала ACK первой, а LB держит delayed ACK ~40 мс -> +40 мс к каждому ответу + # (замер: evidence/load-*.json, раздел «Эксперименты» REPORT.md). + disable_nagle_algorithm = True + server_version = "mtusi-messenger" + sys_version = "" + timeout = CLIENT_SOCKET_TIMEOUT_SECONDS + + def json_body(self): + ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip().lower() + if ctype != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json") + try: + body = json.loads(self._body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise bad_request("Некорректный JSON") + try: + # JSON-escape "\ud800" даёт непарный суррогат: такую строку нельзя закодировать в + # UTF-8 (ни для scrypt, ни для PostgreSQL) — это ошибка клиента, 400 (аудит F08). + json.dumps(body, ensure_ascii=False).encode("utf-8") + except UnicodeEncodeError: + raise bad_request("Строки должны состоять из корректных символов Unicode") + return body + + def current_user(self): + """Только Bearer-токен действующей сессии. Basic и X-User-Id доступа не дают.""" + scheme, _, token = self.headers.get("Authorization", "").partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or not token: + raise unauthorized() + self.token_hash = security.token_hash(token) + user = STORE.get_session_user(self.token_hash) + if user is None: + raise unauthorized() + return user + + def path_uuid(self, value): + parsed = parse_uuid(value) + if parsed is None: + raise bad_request("Неверный UUID в пути") + return parsed + + def _read_body(self): + length = self.headers.get("Content-Length") + if length is None: + self.body_consumed = not self.headers.get("Transfer-Encoding") + return b"" + try: + n = int(length) + except ValueError: + raise bad_request("Неверный Content-Length") + if n < 0: + raise bad_request("Неверный Content-Length") + if n > MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") + deadline = time.monotonic() + JSON_BODY_TIMEOUT_SECONDS + parts, got = [], 0 + try: + while got < n: + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError() + self.connection.settimeout(min(left, CLIENT_SOCKET_TIMEOUT_SECONDS)) + chunk = self.rfile.read1(n - got) + if not chunk: + raise ClientGone() + parts.append(chunk) + got += len(chunk) + except (TimeoutError, socket.timeout): + self.no_drain = True # не дочитываем: соединение будет закрыто + raise ApiError(400, "invalid_request", "Запрос не получен вовремя") + except (ConnectionError, OSError): + raise ClientGone() + finally: + self.connection.settimeout(CLIENT_SOCKET_TIMEOUT_SECONDS) + self.body_consumed = True + return b"".join(parts) + + def _dispatch(self, method): + global _INFLIGHT + with _INFLIGHT_LOCK: + _INFLIGHT += 1 + try: + self._handle(method) + finally: + with _INFLIGHT_LOCK: + _INFLIGHT -= 1 + + def _handle(self, method): + self.request_id = uuid.uuid4().hex + self.extra_headers = {} + self.body_consumed = False + self.body_left = None # сколько байт тела ещё не прочитано (для upload) + self.no_drain = False # тело не дочитывать (срок приёма уже истёк) + self.headers_sent = False + started = time.perf_counter() + parts = urlsplit(self.path) + path = parts.path + func, groups, route = match_route(method, path) + http_method = method if method in _KNOWN_METHODS else "_OTHER" + headers = {} + status, payload, error_type = 500, None, None + HTTP_ACTIVE.add(1, {"http.request.method": http_method}) + # Новый root trace на каждый запрос (context=пустой: traceparent клиента не доверяем). + is_health = route.startswith("/health/") + if is_health and not TRACE_HEALTH: + span = trace.INVALID_SPAN + else: + span = telemetry.tracer.start_span( + "%s %s" % (http_method, route), kind=SpanKind.SERVER, context=telemetry.extract({}), + attributes={"http.request.method": http_method, "http.route": route, "url.scheme": "http", + "network.protocol.version": "1.1", "server.port": PORT, + "messenger.request_id": self.request_id, "service.instance.id": INSTANCE_ID}) + self.trace_id = format(span.get_span_context().trace_id, "032x") if span.get_span_context().is_valid else None + token = trace.use_span(span, end_on_exit=False) + token.__enter__() + try: + try: + try: + self.query = parse_qs(parts.query, keep_blank_values=True, errors="strict") + except UnicodeDecodeError: + raise bad_request("Параметры запроса должны быть в UTF-8") + if func is None: + raise not_found() + if func not in STREAMING_BODY: + self._body = self._read_body() + status, payload = func(self, *groups) + headers = self.extra_headers + except ClientGone: + # Клиент оборвал upload: временный файл уже исчез, S3/БД не тронуты. + self.close_connection = True + status, error_type = None, "client_aborted" + log.info("%s %s -> client aborted upload request_id=%s", method, route, self.request_id, + extra={"event": "http.client_aborted", "request_id": self.request_id, "http.route": route}) + return + except (StorageUnavailable, ObjectMissing): + status, payload, headers = 503, self._error("unavailable", "Хранилище файлов временно недоступно"), {"Retry-After": "1"} + error_type = "storage_unavailable" + except ApiError as err: + status, payload, headers = err.status, self._error(err.code, err.message), err.headers + if err.status >= 500: + error_type = _api_error_type(err) + except (psycopg.OperationalError, PoolTimeout) as exc: + # БД недоступна: честный 503, а не «успех» или 500. + error_type = "db_pool_timeout" if isinstance(exc, PoolTimeout) else "db_unavailable" + log.warning("database unavailable request_id=%s (%s)", self.request_id, type(exc).__name__, + extra={"event": "db.unavailable", "request_id": self.request_id, + "error.type": error_type, "http.route": route}) + status, payload = 503, self._error("unavailable", "Сервис временно недоступен") + except Exception as exc: + error_type = "internal_error" + span.record_exception(exc) + log.exception("unhandled error request_id=%s", self.request_id, + extra={"event": "http.unhandled_error", "request_id": self.request_id, + "http.route": route}) + status, payload = 500, self._error("internal_error", "Внутренняя ошибка сервера") + if payload is STREAMED: + pass + elif self.headers_sent: + # Ошибка после начала потоковой отдачи: статус уже не изменить — рвём соединение. + self.close_connection = True + else: + if not self.body_consumed and not self._drain_rejected_body(): + # Тело не прочитано (ранний отказ upload): соединение нельзя переиспользовать. + self.close_connection = True + self._send(status, payload, headers) + finally: + elapsed = time.perf_counter() - started + attrs = {"http.request.method": http_method, "http.route": route} + if status is not None: + attrs["http.response.status_code"] = status + attrs["http.response.status_class"] = "%dxx" % (status // 100) + span.set_attribute("http.response.status_code", status) + else: + attrs["http.response.status_class"] = "none" + if error_type: + attrs["error.type"] = error_type + span.set_attribute("error.type", error_type) + # Для SERVER span ошибка — только 5xx (4xx — корректный ответ на ошибку клиента). + if status is not None and status >= 500: + span.set_status(Status(StatusCode.ERROR, error_type or str(status))) + HTTP_DURATION.record(elapsed, attrs) + HTTP_ACTIVE.add(-1, {"http.request.method": http_method}) + # В лог не попадают тела запросов, query (filename), пароли, токены и тексты сообщений. + if status is not None and not (is_health and status == 200 and not TRACE_HEALTH): + level = logging.WARNING if status >= 500 else logging.INFO + log.log(level, "%s %s -> %d %.1fms request_id=%s", method, route, status, elapsed * 1000, + self.request_id, + extra={"event": "http.request", "request_id": self.request_id, + "http.request.method": http_method, "http.route": route, + "http.response.status_code": status, "duration_ms": round(elapsed * 1000, 2), + "error.type": error_type or ""}) + token.__exit__(None, None, None) + span.end() + + def _error(self, code, message): + return {"error": {"code": code, "message": message, "request_id": self.request_id}} + + def _trace_header(self): + tid = getattr(self, "trace_id", None) + if tid: + self.send_header("X-Trace-Id", tid) + + def _drain_rejected_body(self): + """Дочитать и выбросить остаток тела отклонённого запроса (буфер 64 KiB). + True — соединение снова в согласованном состоянии.""" + if getattr(self, "no_drain", False): + return False + left = self.body_left + if left is None: + raw = self.headers.get("Content-Length") + if raw is None or not raw.isdigit() or self.headers.get("Transfer-Encoding"): + return False + left = int(raw) + if left > REJECT_DRAIN_BYTES: + return False + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + try: + while left > 0: + if time.monotonic() > deadline: + return False + self.connection.settimeout(max(0.1, min(deadline - time.monotonic(), CLIENT_SOCKET_TIMEOUT_SECONDS))) + chunk = self.rfile.read1(min(UPLOAD_READ_CHUNK, left)) # один recv (аудит F03) + if not chunk: + return False + left -= len(chunk) + except OSError: + return False + finally: + self.connection.settimeout(CLIENT_SOCKET_TIMEOUT_SECONDS) + self.body_consumed = True + return True + + def send_raw_headers(self, status, headers): + """Начало потокового ответа (download): общие заголовки + заданные.""" + self.send_response(status) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-App-Version", APP_VERSION) + self._trace_header() + if DRAINING.is_set() or self.close_connection: + self.close_connection = True + self.send_header("Connection", "close") + for name, value in headers.items(): + self.send_header(name, value) + self.end_headers() + self.headers_sent = True + + def _send(self, status, payload, headers): + self.send_response(status) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-App-Version", APP_VERSION) + self._trace_header() + if DRAINING.is_set() or self.close_connection: + # При остановке закрываем keep-alive: следующий запрос LB уйдёт на живую реплику. + self.close_connection = True + self.send_header("Connection", "close") + for name, value in headers.items(): + self.send_header(name, value) + if payload is None: # 204: без тела + self.end_headers() + return + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_PUT(self): + self._dispatch("PUT") + + def do_PATCH(self): + self._dispatch("PATCH") + + def do_DELETE(self): + self._dispatch("DELETE") + + def log_message(self, fmt, *args): + pass + + def send_error(self, code, message=None, explain=None): + """Ошибки, которые BaseHTTPRequestHandler формирует сам до _dispatch (неизвестный метод, + битая строка запроса, слишком длинный URI/заголовки), — в общем JSON-формате с + X-Request-Id и X-Instance-Id (аудит F17). Неподдержанный метод -> 405 (у стандартного + обработчика было 501 text/html без X-Request-Id). У HEAD тела нет.""" + if code == 501: + code = 405 + # Всегда новый id: на keep-alive соединении атрибут остался бы от прошлого запроса. + self.request_id = uuid.uuid4().hex + self.trace_id = None + payload = {"error": {"code": "invalid_request" if code < 500 else "internal_error", + "message": "Метод не поддерживается" if code == 405 else "Неверный запрос", + "request_id": self.request_id}} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.close_connection = True + try: + self.send_response(code) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-App-Version", APP_VERSION) + if code == 405: + self.send_header("Allow", "GET, POST, DELETE") + self.send_header("Connection", "close") + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if getattr(self, "command", None) != "HEAD": + self.wfile.write(data) + except OSError: + pass + command = getattr(self, "command", None) or "-" + method = command if command in _KNOWN_METHODS else "_OTHER" + HTTP_DURATION.record(0.0, {"http.request.method": method, "http.route": UNMATCHED_ROUTE, + "http.response.status_code": code, + "http.response.status_class": "%dxx" % (code // 100)}) + log.info("%s -> %d (protocol level) request_id=%s", method, code, self.request_id, + extra={"event": "http.protocol_error", "request_id": self.request_id, + "http.request.method": method, "http.response.status_code": code}) + + +# Lab7: mTLS-слушатель API. Клиент обязан предъявить сертификат internal-ca (проверяет OpenSSL), +# а его SAN DNS должен входить в MTLS_ALLOWED_CLIENTS (по умолчанию только "lb"). Сертификат +# другого сервиса того же CA (worker, intruder, ...) отклоняется ДО разбора HTTP. Handshake +# выполняется в потоке соединения (не в accept-цикле): медленный клиент не блокирует остальных. +MTLS_ALLOWED_CLIENTS = frozenset(x.strip() for x in os.environ.get("MTLS_ALLOWED_CLIENTS", "lb").split(",") if x.strip()) +TLS_HANDSHAKE_TIMEOUT_SECONDS = float(os.environ.get("TLS_HANDSHAKE_TIMEOUT_SECONDS", "5")) +HEALTH_HOST = os.environ.get("HEALTH_HOST", "127.0.0.1") +HEALTH_PORT = int(os.environ.get("HEALTH_PORT", "8081")) +MTLS_REJECTS = telemetry.meter.create_counter( + "messenger.mtls.rejected", unit="{connection}", + description="Inbound API connections rejected by mTLS (no cert / bad chain / identity not allowed)") + + +class _TlsRejected(Exception): + pass + + +class Server(ThreadingHTTPServer): + request_queue_size = 1024 + daemon_threads = True + ssl_context = None # None -> plaintext (только loopback health listener) + + def finish_request(self, request, client_address): + if self.ssl_context is None: + return super().finish_request(request, client_address) + request.settimeout(TLS_HANDSHAKE_TIMEOUT_SECONDS) + try: + tls = self.ssl_context.wrap_socket(request, server_side=True) + except (ssl.SSLError, OSError) as exc: + reason = getattr(exc, "reason", None) or type(exc).__name__ + MTLS_REJECTS.add(1, {"reason": "handshake", "error.type": str(reason)[:64]}) + log.warning("mTLS handshake rejected (%s)", reason, + extra={"event": "mtls.rejected", "mtls.reason": str(reason)[:64]}) + return + try: + ok, names = tlsutil.allowed(tls.getpeercert(), MTLS_ALLOWED_CLIENTS) + if not ok: + MTLS_REJECTS.add(1, {"reason": "identity"}) + log.warning("mTLS identity not allowed: %s", ",".join(names) or "-", + extra={"event": "mtls.identity_rejected", "mtls.peer": ",".join(names)[:128]}) + return + self.RequestHandlerClass(tls, client_address, self) + finally: + try: + tls.close() + except OSError: + pass + + def handle_error(self, request, client_address): + """Lab6: вместо многострочного traceback в stderr (вне структурных логов) — событие. + Сброс keep-alive соединения балансировщиком/клиентом между запросами штатен (в lab5 это + давало сотни traceback ConnectionResetError в логах) — его не логируем.""" + exc = sys.exc_info()[1] + if isinstance(exc, (ConnectionResetError, BrokenPipeError, TimeoutError, socket.timeout, ssl.SSLError)): + return + log.error("unhandled server error (%s)", type(exc).__name__, exc_info=True, + extra={"event": "http.server_error", "error.type": type(exc).__name__}) + + +class HealthHandler(Handler): + """Loopback-only listener (127.0.0.1:8081) для Docker healthcheck: только /health/*. + Бизнес-API через него недоступен (404), снаружи контейнера порт не виден.""" + + def _dispatch(self, method): + if method == "GET" and urlsplit(self.path).path in ("/health/live", "/health/ready"): + return super()._dispatch(method) + self.request_id = uuid.uuid4().hex + self.close_connection = True + self._send(404, self._error("not_found", "Не найдено"), {}) + + +def wait_for_schema(timeout_seconds=120): + """Реплика не применяет миграции (это делает отдельный однократный job migrate), + а только ждёт, пока схема БД дойдёт до версии, которую знает этот образ.""" + deadline = time.monotonic() + timeout_seconds + while True: + pending = STORE.pending_migrations(MIGRATIONS_DIR) + if not pending: + return + if time.monotonic() > deadline: + raise SystemExit("schema is not migrated: %s" % ", ".join(pending)) + log.info("waiting for migrations: %s", ", ".join(pending)) + time.sleep(1) + + +def wait_inflight(deadline): + while time.monotonic() < deadline: + with _INFLIGHT_LOCK: + if _INFLIGHT == 0: + return 0 + time.sleep(0.05) + with _INFLIGHT_LOCK: + return _INFLIGHT + + +def graceful_stop(server): + """SIGTERM: 1) ready=503 и Connection: close; 2) DRAIN_SECONDS продолжаем обслуживать, + пока HAProxy не исключит реплику; 3) перестаём принимать соединения; + 4) ждём завершения начатых запросов (не дольше SHUTDOWN_TIMEOUT_SECONDS).""" + DRAINING.set() + log.info("draining: readiness=503 for %.1fs", DRAIN_SECONDS, extra={"event": "app.draining"}) + time.sleep(DRAIN_SECONDS) + server.shutdown() + left = wait_inflight(time.monotonic() + SHUTDOWN_TIMEOUT_SECONDS) + log.info("listener closed, unfinished requests: %d", left, extra={"event": "app.listener_closed"}) + + +def main(): + telemetry.setup("messenger-api") + if PIPELINE_VERSION not in jobs.SUPPORTED_PIPELINE_VERSIONS: + # Аудит F11: другая версия молча давала ready без видимых вариантов. + raise SystemExit("PIPELINE_VERSION=%d is not supported (supported: %s)" + % (PIPELINE_VERSION, ", ".join(map(str, jobs.SUPPORTED_PIPELINE_VERSIONS)))) + STORE.open() + wait_for_schema() + DB_HEALTH.start() + S3_HEALTH.start() + telemetry.observe_pool(STORE.pool, "api") + server = Server((HOST, PORT), Handler) + server.ssl_context = tlsutil.server_context_mtls() + health = Server((HEALTH_HOST, HEALTH_PORT), HealthHandler) + threading.Thread(target=health.serve_forever, name="health-listener", daemon=True).start() + + def stop(signum, frame): + if DRAINING.is_set(): + return + log.info("signal %d received, graceful shutdown", signum, extra={"event": "app.shutdown_requested"}) + threading.Thread(target=graceful_stop, args=(server,), daemon=True).start() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + log.info("listening on %s:%d (mTLS, allowed clients: %s; health %s:%d) instance=%s version=%s env=%s " + "session_ttl=%ds argon2id m=%dKiB t=%d p=%d", HOST, PORT, ",".join(sorted(MTLS_ALLOWED_CLIENTS)), + HEALTH_HOST, HEALTH_PORT, INSTANCE_ID, APP_VERSION, APP_ENV, SESSION_TTL_SECONDS, + security.ARGON2_MEMORY_KIB, security.ARGON2_ITERATIONS, security.ARGON2_PARALLELISM, + extra={"event": "app.started"}) + server.serve_forever() + # serve_forever вернулся из graceful_stop: дождёмся, пока он доведёт in-flight до нуля. + wait_inflight(time.monotonic() + SHUTDOWN_TIMEOUT_SECONDS) + server.server_close() + DB_HEALTH.stop() + S3_HEALTH.stop() + STORE.close() + log.info("stopped", extra={"event": "app.stopped"}) + telemetry.shutdown() + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..df351dc --- /dev/null +++ b/src/store.py @@ -0,0 +1,470 @@ +"""Работа с PostgreSQL: пользователи, чаты, сообщения, сессии, метаданные вложений. + +Lab5: вложение создаётся как pending (скрыто), после записи оригинала в staging одной +транзакцией становится queued вместе с заданием и записью outbox (src/jobs.py). + +Источник истины — база данных. В памяти процесса состояния нет, поэтому +перезапуск API (и самой БД с сохранённым volume) не теряет данные. + +Lab6: каждая публичная операция — CLIENT span «db <операция>» и гистограмма +db.client.operation.duration{db.operation.name}; ожидание соединения пула — отдельная +гистограмма db.client.connection.wait_time и атрибут span. По ним медленная/заблокированная БД +отличается от медленного кода обработчика (spans db длинные vs короткие). +""" +import contextlib +import datetime as dt +import logging +import time +import uuid +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +import jobs +import telemetry +from psycopg_pool import ConnectionPool +from telemetry import db_operation + +log = logging.getLogger("messenger.store") + +EPOCH = dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc) +MIGRATIONS_LOCK_ID = 727001 # произвольный номер advisory-блокировки для миграций + + +class Conflict(Exception): + pass + + +class UnknownMember(Exception): + pass + + +class AttachmentNotFound(Exception): + """Вложение неизвестно, не принято (pending) или принадлежит другому чату -> 404.""" + + +class AttachmentFailed(Exception): + """Вложение в состоянии failed: прикреплять нельзя -> 409.""" + + +def format_ts(value): + """datetime -> RFC 3339 UTC с микросекундами и суффиксом Z.""" + return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def to_micros(value): + delta = value - EPOCH + return (delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds + + +def from_micros(micros): + return EPOCH + dt.timedelta(microseconds=micros) + + +def _user(row): + # Пароль/hash и сессионные данные никогда не входят в User. + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_ts(row["created_at"]), + } + + +def _chat(row): + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in row["member_ids"]], + "created_at": format_ts(row["created_at"]), + } + + +def _message(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_ts(row["created_at"]), + "attachment_ids": [str(a) for a in (row["attachment_ids"] or [])], + } + + +def _attachment(row): + # Наружу не отдаются object_key, bucket и внутренние адреса. + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + # Варианты показываются только у ready (их наличие = готовность уровня). + "variants": list(row["variants"] or []) if row["status"] == "ready" else [], + "error_code": row["error_code"] if row["status"] == "failed" else None, + "created_at": format_ts(row["created_at"]), + "updated_at": format_ts(row["updated_at"]), + } + + +ATTACHMENT_COLUMNS = """a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.status, a.error_code, a.object_key, a.created_at, a.updated_at, + (SELECT json_agg(json_build_object('name', v.name, 'content_type', v.content_type, + 'size_bytes', v.size_bytes, 'width', v.width, 'height', v.height, 'sha256', v.sha256) + ORDER BY array_position(ARRAY['thumbnail','crop','optimized'], v.name)) + FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variants""" + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + (SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma + WHERE ma.message_id = m.id) AS attachment_ids""" + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + (SELECT array_agg(m.user_id ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) AS member_ids""" + + +class InstrumentedPool(ConnectionPool): + """ConnectionPool, который измеряет ожидание соединения (насыщение пула видно в метриках).""" + + @contextlib.contextmanager + def connection(self, timeout=None): + started = time.perf_counter() + outcome = {} + try: + conn = self.getconn(timeout=timeout) + except Exception as exc: + outcome["error.type"] = type(exc).__name__ + raise + finally: + waited = time.perf_counter() - started + telemetry.db_wait.record(waited, dict({"db.client.connection.pool.name": "api", + "db.system.name": telemetry.DB_SYSTEM}, **outcome)) + span = telemetry.trace.get_current_span() + if span.is_recording(): + span.set_attribute("db.client.connection.wait_time_ms", round(waited * 1000, 3)) + try: + with conn: + yield conn + finally: + self.putconn(conn) + + +class Store: + def __init__(self, conninfo, pool_size=10, pool_timeout=3): + # check: перед выдачей соединение проверяется. После перезапуска PostgreSQL + # «мёртвые» соединения отбрасываются, и запрос не получает ложный 503. + self.pool = InstrumentedPool( + conninfo, min_size=1, max_size=pool_size, timeout=pool_timeout, + kwargs={"row_factory": dict_row}, open=False, + check=ConnectionPool.check_connection) + + # --- запуск --- + def open(self, wait_seconds=60): + """Ждём готовности БД (она может стартовать дольше API) и открываем пул.""" + self.pool.open() + deadline = time.monotonic() + wait_seconds + while True: + try: + with self.pool.connection() as conn: + conn.execute("SELECT 1") + return + except Exception as exc: + if time.monotonic() > deadline: + raise + log.info("database is not ready yet (%s), retrying", type(exc).__name__) + time.sleep(1) + + def close(self): + self.pool.close() + + def migrate(self, directory): + """Применяет *.sql по порядку имён; каждая миграция — одна транзакция, один раз.""" + files = sorted(Path(directory).glob("*.sql")) + with self.pool.connection() as conn: + # Advisory-блокировка (сессионная): параллельный повторный запуск job не устроит гонку. + conn.autocommit = True + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK_ID,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())""") + done = {r["version"] for r in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + # Миграция и её отметка — одна транзакция: либо целиком, либо никак. + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + log.info("migration applied: %s", path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK_ID,)) + conn.autocommit = False + + def pending_migrations(self, directory): + """Миграции из каталога образа, ещё не применённые к БД (API их НЕ применяет).""" + files = sorted(p.name for p in Path(directory).glob("*.sql")) + with self.pool.connection() as conn: + exists = conn.execute("SELECT to_regclass('schema_migrations') IS NOT NULL AS ok").fetchone()["ok"] + done = {r["version"] for r in conn.execute("SELECT version FROM schema_migrations")} if exists else set() + return [f for f in files if f not in done] + + @db_operation("ping") + def ping(self): + with self.pool.connection() as conn: + conn.execute("SELECT 1") + + # --- пользователи --- + @db_operation("create_user") + def create_user(self, username, display_name, password_hash): + try: + with self.pool.connection() as conn: + row = conn.execute( + """INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, clock_timestamp()) + RETURNING id, username, display_name, created_at""", + (uuid.uuid4(), username, display_name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + # Уникальность гарантирует сама БД — гонка двух регистраций безопасна. + raise Conflict() + return _user(row) + + @db_operation("get_credentials") + def get_credentials(self, username): + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,)).fetchone() + if row is None: + return None, None + return _user(row), row["password_hash"] + + @db_operation("rehash_password") + def update_password_hash(self, user_id, old_hash, new_hash): + """Lab7 rehash: заменяем только тот хеш, который проверяли (гонка двух входов безопасна).""" + with self.pool.connection() as conn: + conn.execute("UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s", + (new_hash, user_id, old_hash)) + + # --- сессии --- + @db_operation("create_session") + def create_session(self, user_id, token_hash, ttl_seconds): + with self.pool.connection() as conn: + row = conn.execute( + """INSERT INTO sessions (token_hash, user_id, created_at, expires_at) + VALUES (%s, %s, now(), now() + make_interval(secs => %s)) + RETURNING expires_at""", + (token_hash, user_id, ttl_seconds)).fetchone() + return format_ts(row["expires_at"]) + + @db_operation("get_session_user") + def get_session_user(self, token_hash): + """Пользователь действующей сессии или None (неизвестна/истекла/отозвана).""" + with self.pool.connection() as conn: + row = conn.execute( + """SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > now()""", + (token_hash,)).fetchone() + return _user(row) if row else None + + @db_operation("revoke_session") + def revoke_session(self, token_hash): + with self.pool.connection() as conn: + conn.execute( + "UPDATE sessions SET revoked_at = now() WHERE token_hash = %s AND revoked_at IS NULL", + (token_hash,)) + + # --- чаты --- + @db_operation("create_chat") + def create_chat(self, title, member_ids): + """Чат и его участники создаются в одной транзакции (атомарно).""" + chat_id = uuid.uuid4() + with self.pool.connection() as conn: + found = conn.execute( + "SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (member_ids,)).fetchone()["n"] + if found != len(member_ids): + raise UnknownMember() + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, clock_timestamp())", + (chat_id, title)) + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + [(chat_id, m, i) for i, m in enumerate(member_ids)]) + row = conn.execute("SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id = %s", (chat_id,)).fetchone() + return _chat(row) + + @db_operation("get_chat_for") + def get_chat_for(self, user_id, chat_id): + """Чат, если пользователь — участник; иначе None (чужой и несуществующий неразличимы).""" + with self.pool.connection() as conn: + row = conn.execute( + "SELECT " + CHAT_COLUMNS + """ FROM chats c + WHERE c.id = %s AND EXISTS ( + SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = %s)""", + (chat_id, user_id)).fetchone() + return _chat(row) if row else None + + @db_operation("list_chats") + def list_chats(self, user_id, after, limit): + params = [user_id] + where = "" + if after: + where = "AND (c.created_at, c.id) > (%s, %s::uuid)" + params += [from_micros(after[0]), after[1]] + params.append(limit + 1) # +1 запись: узнаём, есть ли следующая страница + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT " + CHAT_COLUMNS + """ FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE true """ + where + " ORDER BY c.created_at, c.id LIMIT %s", params).fetchall() + return self._page(rows, limit, _chat) + + # --- сообщения --- + @db_operation("add_message") + def add_message(self, user_id, chat_id, text, attachment_ids, idem_key, request_hash, idem_ttl_seconds): + """Идемпотентная отправка. Возвращает (message, replayed) или None, если чата нет/он чужой. + + Всё в ОДНОЙ транзакции: проверка доступа, поиск ключа, вставка сообщения и запись + ключа. Либо фиксируется и сообщение, и ключ, либо ничего (транзиентная ошибка + не запоминается как результат — клиент повторит с тем же ключом). + Конкурентные запросы с одним ключом через разные реплики сериализуются + блокировкой строки чата (FOR UPDATE), второй после COMMIT первого видит ключ. + PK таблицы ключей — страховка: даже без блокировки дубль не зафиксируется. + Lab4: в той же транзакции проверяется, что все attachment_ids — ready-вложения + ЭТОГО чата (FOR SHARE: строка не исчезнет до COMMIT), и пишутся ссылки. + """ + with self.pool.connection() as conn: + # FOR UPDATE по строке чата: вставки в один чат идут по очереди, поэтому + # created_at растёт в порядке фиксации и keyset-курсор не пропускает записи. + allowed = conn.execute( + """SELECT 1 FROM chats c WHERE c.id = %s AND EXISTS ( + SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = %s) + FOR UPDATE OF c""", (chat_id, user_id)).fetchone() + if allowed is None: + return None + existing = conn.execute( + """SELECT k.request_hash, k.expires_at > now() AS alive, """ + MESSAGE_COLUMNS + """ + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = %s AND k.chat_id = %s AND k.idem_key = %s""", + (user_id, chat_id, idem_key)).fetchone() + if existing is not None: + if existing["alive"]: + if bytes(existing["request_hash"]) != request_hash: + raise Conflict() + return _message(existing), True + # Ключ старше TTL (24 ч) — освобождаем его и обрабатываем как новый. + conn.execute( + "DELETE FROM idempotency_keys WHERE user_id = %s AND chat_id = %s AND idem_key = %s", + (user_id, chat_id, idem_key)) + if attachment_ids: + # Lab5: queued/processing/ready можно прикреплять; failed — 409; + # pending (upload ещё не принят) — как неизвестное. + found = conn.execute( + """SELECT id, status FROM attachments + WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status <> 'pending' + FOR SHARE""", (attachment_ids, chat_id)).fetchall() + if len(found) != len(attachment_ids): + # Чужой чат, неизвестный или непринятый id — одинаково 404. + raise AttachmentNotFound() + if any(r["status"] == "failed" for r in found): + raise AttachmentFailed() + row = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, clock_timestamp()) + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid.uuid4(), chat_id, user_id, text)).fetchone() + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES (%s, %s, %s)""", + [(row["id"], a, i) for i, a in enumerate(attachment_ids)]) + row = dict(row, attachment_ids=list(attachment_ids)) + conn.execute( + """INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id, expires_at) + VALUES (%s, %s, %s, %s, %s, now() + make_interval(secs => %s))""", + (user_id, chat_id, idem_key, request_hash, row["id"], idem_ttl_seconds)) + return _message(row), False + + @db_operation("list_messages") + def list_messages(self, chat_id, after, limit): + params = [chat_id] + where = "" + if after: + where = "AND (m.created_at, m.id) > (%s, %s::uuid)" + params += [from_micros(after[0]), after[1]] + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT " + MESSAGE_COLUMNS + " FROM messages m WHERE m.chat_id = %s " + + where + " ORDER BY m.created_at, m.id LIMIT %s", params).fetchall() + return self._page(rows, limit, _message) + + # --- вложения --- + @db_operation("create_pending_attachment") + def create_pending_attachment(self, att_id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, width, height, object_key, staging_key, pipeline_version): + """Шаг 1: намерение фиксируется ДО PUT в staging (как в lab4). Строка pending + API не показывает; брошенную (крах до постановки в очередь) убирает janitor.""" + with self.pool.connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key, staging_key, pipeline_version, status, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending', now(), now())""", + (att_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, staging_key, pipeline_version)) + + @db_operation("enqueue_attachment") + def enqueue_attachment(self, att_id, pipeline_version, max_attempts, before_commit=None): + """Шаг 3 (после PUT в staging): ОДНА транзакция pending -> queued + jobs + outbox. + После COMMIT работа принята: даже если API сразу упадёт, relay опубликует задание. + None — строку уже убрал janitor (ответ 503).""" + with self.pool.connection() as conn: + job_id = jobs.enqueue(conn, att_id, pipeline_version, max_attempts) + if job_id is None: + return None + if before_commit: + before_commit() + row = conn.execute("SELECT " + ATTACHMENT_COLUMNS + " FROM attachments a WHERE a.id = %s", + (att_id,)).fetchone() + return _attachment(row) + + @db_operation("delete_pending_attachment") + def delete_pending_attachment(self, att_id): + """Компенсация: убрать непринятое намерение (staging-объект удалён/не создан).""" + with self.pool.connection() as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'", (att_id,)) + + @db_operation("get_attachment_for") + def get_attachment_for(self, user_id, att_id): + """(attachment, object_key) для участника чата; иначе None. Pending не виден.""" + with self.pool.connection() as conn: + row = conn.execute( + "SELECT " + ATTACHMENT_COLUMNS + """ FROM attachments a + WHERE a.id = %s AND a.status <> 'pending' AND EXISTS ( + SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = %s)""", + (att_id, user_id)).fetchone() + return (_attachment(row), row["object_key"]) if row else None + + @db_operation("get_variant_key") + def get_variant_key(self, att_id, name): + """object_key варианта текущей версии pipeline (или None).""" + with self.pool.connection() as conn: + row = conn.execute( + """SELECT v.object_key FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id + WHERE v.attachment_id = %s AND v.name = %s AND v.pipeline_version = a.pipeline_version""", + (att_id, name)).fetchone() + return row["object_key"] if row else None + + @staticmethod + def _page(rows, limit, convert): + has_more = len(rows) > limit + rows = rows[:limit] + last = (to_micros(rows[-1]["created_at"]), str(rows[-1]["id"])) if rows and has_more else None + return [convert(r) for r in rows], last diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..f6bf717 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,492 @@ +"""Lab6: наблюдаемость всех процессов (API, relay, worker, janitor) через OpenTelemetry SDK. + +Три сигнала уходят по OTLP/HTTP (protobuf) в OTel Collector (OTEL_EXPORTER_OTLP_ENDPOINT): + traces -> BatchSpanProcessor (ограниченная очередь, сброс раз в 1 с) -> Collector -> Tempo + metrics -> PeriodicExportingMetricReader (раз в OTEL_METRIC_EXPORT_INTERVAL) -> Collector -> Prometheus + logs -> logging.Handler -> BatchLogRecordProcessor -> Collector -> Loki +и параллельно — структурный JSON в stdout (`docker compose logs`), с trace_id/span_id. + +Принципы: + * телеметрия никогда не ломает бизнес-путь: все очереди SDK ограничены (max_queue_size), + экспорт — с таймаутом в фоновых потоках; недоступный Collector = потерянная телеметрия, + а не ошибка запроса. Потери считает сам SDK: otel.sdk.processor.*.processed{error.type} + и otel.sdk.exporter.*.exported{error.type} (meter_provider передаётся процессорам/экспортёрам); + * в атрибуты МЕТРИК идут только низкокардинальные значения (метод, шаблон маршрута, класс + статуса, тип/исход задания, имя операции). ID пользователей/чатов/сообщений/вложений, + request_id, raw URL, filename и тексты — никогда; + * в трассы/логи допускаются технические ID синтетических учебных данных (job_id, + attachment_id, request_id) — по ним upload связывается с заданием. Пароли, Authorization, + токены, тела сообщений, байты изображений, URL с паролями — запрещены (см. redact()); + * OTEL_SDK_DISABLED=true выключает экспорт полностью (no-op провайдеры), stdout-логи остаются. + +Resource: service.name (messenger-api | -worker | -relay | -janitor), service.version +(= APP_VERSION образа), service.instance.id (= hostname контейнера = X-Instance-Id), +deployment.environment.name (из OTEL_RESOURCE_ATTRIBUTES, по умолчанию lab). +""" +import contextlib +import datetime as dt +import functools +import json +import logging +import os +import re +import socket +import sys +import threading +import time + +from opentelemetry import metrics, propagate, trace +from opentelemetry.trace import SpanKind, Status, StatusCode + +SERVICE_NAME = os.environ.get("OTEL_SERVICE_NAME", "messenger-app") +SERVICE_VERSION = os.environ.get("APP_VERSION", "dev") +from instance import INSTANCE_ID # lab7: случайный alias, не hostname +ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true" + +tracer = trace.get_tracer("messenger", SERVICE_VERSION) +meter = metrics.get_meter("messenger", SERVICE_VERSION) + +_setup_done = False +_setup_lock = threading.Lock() +_providers = [] + +# -------------------------------------------------------------------------- +# Гистограммы: границы под ожидаемые задержки (секунды). p50/p95/p99 считаются в Prometheus +# из buckets (histogram_quantile), а не из средних. +# -------------------------------------------------------------------------- +# 3.5 и 5.5 с — рядом с PoolTimeout (3 с) и statement_timeout (5 с): без них ответы по таймауту +# попадали в (5, 10] и p95 линейной интерполяцией показывал ~9.75 с вместо ~5 с. +HTTP_BUCKETS = (0.002, 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1.0, + 1.5, 2.5, 3.5, 5.0, 5.5, 7.5, 10.0, 30.0, 60.0) +DB_BUCKETS = (0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 3.5, 5.0, 5.5, + 7.5, 10.0) +JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0) +READY_BUCKETS = (0.25, 0.5, 1.0, 1.5, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0, 300.0, + 600.0, 1800.0, 3600.0) + + +def _env_float(name, default): + try: + return float(os.environ.get(name, str(default))) + except ValueError: + return float(default) + + +def _env_int(name, default): + try: + return int(os.environ.get(name, str(default))) + except ValueError: + return int(default) + + +# -------------------------------------------------------------------------- +# Защита от утечек в логах (defense in depth: код и так не логирует секреты/тела). +# -------------------------------------------------------------------------- +_REDACT = [ + (re.compile(r"(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{6,}"), r"\1 [REDACTED]"), + (re.compile(r"(?i)(authorization\s*[:=]\s*)\S+"), r"\1[REDACTED]"), + (re.compile(r"([a-z][a-z0-9+.-]*://)[^/\s:@]+:[^@\s/]+@"), r"\1[REDACTED]@"), + (re.compile(r"(?i)(password|passwd|secret|token)(\"?\s*[:=]\s*\"?)[^\s\",}]+"), r"\1\2[REDACTED]"), + (re.compile(r"(?i)(X-Amz-Signature|X-Amz-Credential|Signature)=[^&\s]+"), r"\1=[REDACTED]"), +] + + +def redact(text): + if not isinstance(text, str): + return text + for pattern, repl in _REDACT: + text = pattern.sub(repl, text) + return text + + +_STD_ATTRS = set(vars(logging.LogRecord("", 0, "", 0, "", None, None))) | {"message", "asctime", "taskName"} + + +class EventFilter(logging.Filter): + """У каждой записи есть поле event (стабильное имя события) и отредактированный текст.""" + + def filter(self, record): + if not getattr(record, "event", None): + record.event = "log" + if record.args: + try: + record.msg = record.getMessage() + record.args = None + except Exception: + pass + record.msg = redact(record.msg) + return True + + +class JsonFormatter(logging.Formatter): + """Структурный лог в stdout: timestamp, severity, service.name, event, trace_id, span_id.""" + + def format(self, record): + out = { + "timestamp": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds"), + "severity": record.levelname, + "service.name": SERVICE_NAME, + "service.instance.id": INSTANCE_ID, + "logger": record.name, + "event": getattr(record, "event", "log"), + "message": record.getMessage(), + } + ctx = trace.get_current_span().get_span_context() + if ctx.is_valid: + out["trace_id"] = format(ctx.trace_id, "032x") + out["span_id"] = format(ctx.span_id, "016x") + for key, value in vars(record).items(): + if key not in _STD_ATTRS and key not in out and not key.startswith("_") and key != "event": + out[key] = value if isinstance(value, (int, float, bool)) or value is None else redact(str(value)) + if record.exc_info: + out["exception.type"] = record.exc_info[0].__name__ if record.exc_info[0] else None + out["exception.stacktrace"] = redact(self.formatException(record.exc_info)) + return json.dumps(out, ensure_ascii=False) + + +class _NotOtelInternals(logging.Filter): + """Логи самого SDK/экспортёра не отправляем через тот же экспортёр (иначе при недоступном + Collector каждая ошибка экспорта порождала бы новую запись в очереди экспорта).""" + + def filter(self, record): + return not (record.name.startswith("opentelemetry") or record.name.startswith("urllib3.connectionpool")) + + +def configure_logging(stream=None): + root = logging.getLogger() + for h in list(root.handlers): + root.removeHandler(h) + handler = logging.StreamHandler(stream or sys.stdout) + handler.setFormatter(JsonFormatter()) + handler.addFilter(EventFilter()) + root.addHandler(handler) + root.setLevel(logging.INFO) + logging.getLogger("pika").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + return handler + + +# -------------------------------------------------------------------------- +# Настройка SDK +# -------------------------------------------------------------------------- +def setup(service_name=None): + """Идемпотентно: логирование (stdout JSON), затем — если не выключено — провайдеры OTel.""" + global _setup_done, SERVICE_NAME + with _setup_lock: + if _setup_done: + return + _setup_done = True + if service_name and not os.environ.get("OTEL_SERVICE_NAME"): + SERVICE_NAME = service_name + configure_logging() + if not ENABLED: + logging.getLogger("telemetry").info("OpenTelemetry export disabled (OTEL_SDK_DISABLED=true)", + extra={"event": "telemetry.disabled"}) + return + _setup_sdk() + + +def _setup_sdk(): + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import Histogram, MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # Resource.create() добавляет OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab) + # и атрибуты telemetry.sdk.*; явные значения ниже важнее. + resource = Resource.create({ + "service.name": SERVICE_NAME, + "service.version": SERVICE_VERSION, + "service.instance.id": INSTANCE_ID, + }) + if "deployment.environment.name" not in resource.attributes: + resource = resource.merge(Resource({"deployment.environment.name": os.environ.get("APP_ENV", "lab")})) + timeout = _env_float("OTEL_EXPORTER_OTLP_TIMEOUT_SECONDS", 5) + + def views(): + out = [] + for name, buckets in (("http.server.request.duration", HTTP_BUCKETS), + ("db.client.operation.duration", DB_BUCKETS), + ("db.client.connection.wait_time", DB_BUCKETS), + ("messenger.s3.operation.duration", DB_BUCKETS), + ("messenger.job.duration", JOB_BUCKETS), + ("messenger.job.queue_wait", READY_BUCKETS), + ("messenger.attachment.time_to_ready", READY_BUCKETS), + ("messenger.outbox.publish_delay", READY_BUCKETS)): + out.append(View(instrument_type=Histogram, instrument_name=name, + aggregation=ExplicitBucketHistogramAggregation(buckets))) + return out + + # 1) Метрики первыми: их meter_provider получают процессоры/экспортёры других сигналов + # и считают собственные потери (queue_full, ошибки экспорта). + metric_exporter = OTLPMetricExporter(timeout=timeout) + reader = PeriodicExportingMetricReader( + metric_exporter, + export_interval_millis=_env_int("OTEL_METRIC_EXPORT_INTERVAL", 10000), + export_timeout_millis=_env_int("OTEL_METRIC_EXPORT_TIMEOUT", 5000)) + meter_provider = MeterProvider(resource=resource, metric_readers=[reader], views=views()) + metrics.set_meter_provider(meter_provider) + + # 2) Трассы. Сэмплер по умолчанию parentbased_always_on; меняется OTEL_TRACES_SAMPLER(_ARG). + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(timeout=timeout, meter_provider=meter_provider), + max_queue_size=_env_int("OTEL_BSP_MAX_QUEUE_SIZE", 2048), + schedule_delay_millis=_env_int("OTEL_BSP_SCHEDULE_DELAY", 1000), + max_export_batch_size=_env_int("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", 512), + export_timeout_millis=_env_int("OTEL_BSP_EXPORT_TIMEOUT", 5000), + meter_provider=meter_provider)) + trace.set_tracer_provider(tracer_provider) + + # 3) Логи: тот же формат событий (event, request_id, ...) как атрибуты записи; trace_id/span_id + # берутся из текущего контекста автоматически. + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(timeout=timeout, meter_provider=meter_provider), + max_queue_size=_env_int("OTEL_BLRP_MAX_QUEUE_SIZE", 2048), + schedule_delay_millis=_env_int("OTEL_BLRP_SCHEDULE_DELAY", 1000), + max_export_batch_size=_env_int("OTEL_BLRP_MAX_EXPORT_BATCH_SIZE", 512), + export_timeout_millis=_env_int("OTEL_BLRP_EXPORT_TIMEOUT", 5000), + meter_provider=meter_provider)) + set_logger_provider(logger_provider) + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.addFilter(EventFilter()) + otel_handler.addFilter(_NotOtelInternals()) + logging.getLogger().addHandler(otel_handler) + + _providers.extend([tracer_provider, logger_provider, meter_provider]) + _register_resource_gauges() + logging.getLogger("telemetry").info( + "OpenTelemetry enabled endpoint=%s protocol=%s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "-"), + os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"), extra={"event": "telemetry.enabled"}) + + +def shutdown(timeout_millis=3000): + """Сбросить буферы при штатной остановке. Общий срок timeout_millis на ВСЕ провайдеры: + при недоступном Collector экспортёр повторяет попытки, и без срока остановка API + затянулась бы дольше stop_grace_period (замер: ~14 с без ограничения).""" + providers = list(_providers) + _providers.clear() + threads = [] + for p in providers: + t = threading.Thread(target=_quiet_shutdown, args=(p,), daemon=True) + t.start() + threads.append(t) + deadline = time.monotonic() + timeout_millis / 1000.0 + for t in threads: + t.join(max(0.0, deadline - time.monotonic())) + + +def _quiet_shutdown(provider): + try: + provider.shutdown() + except Exception: + pass + + +# -------------------------------------------------------------------------- +# Ресурсы контейнера/процесса (cgroup v2 своего контейнера, /proc/self): saturation CPU/RAM. +# Включают подпроцессы (проба JPEG в API, transform в worker). +# -------------------------------------------------------------------------- +def _read(path): + try: + with open(path, encoding="ascii") as f: + return f.read().strip() + except OSError: + return None + + +def _cgroup_cpu_seconds(): + raw = _read("/sys/fs/cgroup/cpu.stat") + if not raw: + return None + for line in raw.splitlines(): + k, _, v = line.partition(" ") + if k == "usage_usec": + return int(v) / 1e6 + return None + + +def _cgroup_cpu_limit(): + raw = _read("/sys/fs/cgroup/cpu.max") + if not raw: + return None + quota, _, period = raw.partition(" ") + if quota == "max": + return float(os.cpu_count() or 1) + return int(quota) / int(period or 100000) + + +def _process_rss_bytes(): + raw = _read("/proc/self/status") + if raw: + for line in raw.splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) * 1024 + return None + + +def _register_resource_gauges(): + def cpu_time(options): + v = _cgroup_cpu_seconds() + if v is not None: + yield metrics.Observation(v) + + def mem_usage(options): + v = _read("/sys/fs/cgroup/memory.current") + if v and v.isdigit(): + yield metrics.Observation(int(v)) + + def mem_limit(options): + v = _read("/sys/fs/cgroup/memory.max") + if v and v.isdigit(): + yield metrics.Observation(int(v)) + + def cpu_limit(options): + v = _cgroup_cpu_limit() + if v is not None: + yield metrics.Observation(v) + + def rss(options): + v = _process_rss_bytes() + if v is not None: + yield metrics.Observation(v) + + def threads(options): + yield metrics.Observation(threading.active_count()) + + meter.create_observable_counter("container.cpu.time", [cpu_time], unit="s", + description="CPU time of this container (cgroup v2 cpu.stat usage_usec)") + meter.create_observable_gauge("container.memory.usage", [mem_usage], unit="By", + description="Memory of this container (cgroup v2 memory.current)") + meter.create_observable_gauge("container.memory.limit", [mem_limit], unit="By", + description="Memory limit of this container (cgroup v2 memory.max)") + meter.create_observable_gauge("container.cpu.limit", [cpu_limit], unit="{cpu}", + description="CPU quota of this container in cores (cgroup v2 cpu.max)") + meter.create_observable_gauge("process.memory.usage", [rss], unit="By", + description="Resident memory of the main process (VmRSS)") + meter.create_observable_gauge("process.thread.count", [threads], unit="{thread}", + description="Python threads of the main process") + + +# -------------------------------------------------------------------------- +# Общие инструменты +# -------------------------------------------------------------------------- +DB_SYSTEM = "postgresql" +DB_NAMESPACE = os.environ.get("POSTGRES_DB", "messenger") + +db_duration = meter.create_histogram( + "db.client.operation.duration", unit="s", + description="Duration of a logical database operation (pool wait included)") +db_wait = meter.create_histogram( + "db.client.connection.wait_time", unit="s", + description="Time spent waiting for a connection from the pool") +s3_duration = meter.create_histogram( + "messenger.s3.operation.duration", unit="s", description="Duration of an S3 request") + + +def current_trace_ids(): + ctx = trace.get_current_span().get_span_context() + if not ctx.is_valid: + return None, None + return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") + + +def has_active_span(): + return trace.get_current_span().get_span_context().is_valid + + +def error_type_of(exc): + return type(exc).__name__ + + +@contextlib.contextmanager +def child_span(name, kind=SpanKind.INTERNAL, attributes=None, root=False): + """Span только внутри уже начатой трассы (запрос/задание). Фоновые циклы (health-пинги, + heartbeat аренды, опрос outbox) без родителя трасс не порождают — только метрики.""" + if not root and not has_active_span(): + yield None + return + with tracer.start_as_current_span(name, kind=kind, attributes=attributes or {}, + record_exception=False, set_status_on_exception=False) as span: + try: + yield span + except BaseException as exc: + span.set_attribute("error.type", error_type_of(exc)) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + raise + + +def db_operation(op): + """Декоратор логической операции БД: CLIENT span «db » (если есть родитель) и + гистограмма db.client.operation.duration{db.operation.name, error.type}.""" + def wrap(func): + @functools.wraps(func) + def inner(*args, **kwargs): + started = time.perf_counter() + attrs = {"db.system.name": DB_SYSTEM, "db.operation.name": op} + err = None + try: + with child_span("db " + op, SpanKind.CLIENT, + dict(attrs, **{"db.namespace": DB_NAMESPACE, "server.address": "db"})): + return func(*args, **kwargs) + except BaseException as exc: + err = error_type_of(exc) + raise + finally: + if err: + attrs["error.type"] = err + db_duration.record(time.perf_counter() - started, attrs) + return inner + return wrap + + +def observe_pool(pool, pool_name): + """Наблюдаемые метрики пула psycopg_pool (semantic conventions db.client.connection.*).""" + base = {"db.client.connection.pool.name": pool_name, "db.system.name": DB_SYSTEM} + + def count(options): + s = pool.get_stats() + size, available = s.get("pool_size", 0), s.get("pool_available", 0) + yield metrics.Observation(max(0, size - available), dict(base, **{"db.client.connection.state": "used"})) + yield metrics.Observation(available, dict(base, **{"db.client.connection.state": "idle"})) + + def maximum(options): + yield metrics.Observation(pool.get_stats().get("pool_max", 0), base) + + def pending(options): + yield metrics.Observation(pool.get_stats().get("requests_waiting", 0), base) + + def timeouts(options): + yield metrics.Observation(pool.get_stats().get("requests_errors", 0), base) + + meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}", + description="Connections in the pool by state (used/idle)") + meter.create_observable_up_down_counter("db.client.connection.max", [maximum], unit="{connection}", + description="Pool max size") + meter.create_observable_up_down_counter("db.client.connection.pending_requests", [pending], + unit="{request}", description="Requests waiting for a connection") + meter.create_observable_counter("db.client.connection.timeouts", [timeouts], unit="{timeout}", + description="Pool requests that failed (timeout waiting for a connection)") + + +# -------------------------------------------------------------------------- +# W3C Trace Context: перенос через outbox/AMQP +# -------------------------------------------------------------------------- +def inject_dict(): + """{'traceparent': ..., 'tracestate': ...} текущего контекста (пусто, если span нет).""" + carrier = {} + propagate.inject(carrier) + return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate") and v} + + +def extract(carrier): + return propagate.extract(carrier or {}) diff --git a/src/tlsutil.py b/src/tlsutil.py new file mode 100644 index 0000000..06601a1 --- /dev/null +++ b/src/tlsutil.py @@ -0,0 +1,55 @@ +"""Lab7: TLS-контексты и проверка identity по SAN. + +Все внутренние связи проверяют сертификат сервера (CA internal-ca + hostname = имя сервиса). +Клиентская identity процесса — его leaf-сертификат (SAN DNS = имя сервиса Compose); ключ +приходит Docker secret-ом /run/secrets/service.key и не покидает контейнер. + +Отключить проверку сертификата конфигурацией нельзя: CERT_NONE/check_hostname=False здесь не +встречаются, «fallback на plaintext» нет — без файла CA процесс падает при старте. +""" +import os +import ssl + +CA_FILE = os.environ.get("TLS_CA_FILE", "/run/secrets/internal-ca.crt") +CERT_FILE = os.environ.get("TLS_CERT_FILE", "/run/secrets/service.crt") +KEY_FILE = os.environ.get("TLS_KEY_FILE", "/run/secrets/service.key") + + +def has_identity(): + return os.path.exists(CERT_FILE) and os.path.exists(KEY_FILE) + + +def client_context(with_identity=True): + """Клиент: TLS 1.2+, проверка цепочки и hostname; при наличии — свой client certificate.""" + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=CA_FILE) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = True + if with_identity and has_identity(): + ctx.load_cert_chain(CERT_FILE, KEY_FILE) + return ctx + + +def server_context_mtls(): + """Сервер API: обязательный client certificate от internal-ca (цепочка проверяется OpenSSL). + Разрешённую identity (SAN) проверяет вызывающий код — доверенный CA ещё не означает «это LB».""" + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(CERT_FILE, KEY_FILE) + ctx.load_verify_locations(cafile=CA_FILE) + ctx.verify_mode = ssl.CERT_REQUIRED + # Без session tickets: каждое новое соединение заново предъявляет и проверяет сертификат + # (важно для отзыва/ротации identity: resumption не продлевает доступ старому сертификату). + ctx.options |= ssl.OP_NO_TICKET + ctx.set_alpn_protocols(["http/1.1"]) + return ctx + + +def peer_dns_names(cert): + """SAN DNS из getpeercert() (dict). CN не используется — только subjectAltName.""" + return [value for kind, value in (cert or {}).get("subjectAltName", ()) if kind == "DNS"] + + +def allowed(cert, allowlist): + names = peer_dns_names(cert) + return bool(names) and any(n in allowlist for n in names), names diff --git a/src/transform.py b/src/transform.py new file mode 100644 index 0000000..970a3d2 --- /dev/null +++ b/src/transform.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Преобразование одного изображения в отдельном ПРОЦЕССЕ с жёсткими лимитами. + + python transform.py + -> stdout: JSON {"variants": [...]} ; код выхода — класс ошибки. + +Worker запускает его через subprocess с таймаутом JOB_TIMEOUT_SECONDS (wall clock), а сам +процесс до импорта Pillow ограничивает себе адресное пространство (RLIMIT_AS) и CPU-время +(RLIMIT_CPU). «Бомба», которую не поймали проверки заголовка, убивает только этот процесс, +а не worker с его соединениями и арендами. + +Коды выхода: 0 — успех; 3 — invalid_image; 4 — image_too_large; 5 — resource_limit +(MemoryError). Убит сигналом (SIGXCPU/SIGKILL) -> resource_limit; таймаут — решает worker. + +Варианты (contracts/README.md, «Фоновая обработка»): + thumbnail — JPEG q=85, вписан в 256x256 с сохранением пропорций, без увеличения; + размеры считаются целочисленно с округлением вниз, минимум 1 px + (320x200 -> 256x160); EXIF orientation применяется; alpha -> на белый фон; + crop — JPEG q=85, центрированный квадрат 128x128 (маленькие исходники увеличиваются); + optimized — WebP q=80 method=4, вписан в 1280x1280 без увеличения; alpha сохраняется. +Во все варианты НЕ переносятся EXIF/XMP/ICC/комментарии. Одного «не передавать exif=» мало: +Pillow при save() берёт часть метаданных из im.info (так JPEG-комментарий доходил до +thumbnail и crop — аудит F07). Поэтому перед сохранением info очищается полностью, а тест +проверяет сырые байты результата (маркеры COM/APP1/APP2, чанки EXIF/XMP/ICCP). +Прозрачность: RGBA/LA/PA и ЛЮБОЙ режим с tRNS (P, RGB, L — аудит F06). Цвет считается sRGB. +""" +import json +import os +import resource +import sys + +MEMORY_LIMIT_MB = int(os.environ.get("TRANSFORM_MEMORY_LIMIT_MB", "768")) +CPU_LIMIT_SECONDS = int(os.environ.get("TRANSFORM_CPU_SECONDS", "30")) +THUMB = 256 +CROP = 128 +OPTIMIZED = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +EXIT_INVALID, EXIT_TOO_LARGE, EXIT_RESOURCE = 3, 4, 5 + + +def fit_within(w, h, box): + """Вписать (w, h) в box x box без увеличения; целочисленно, вниз, минимум 1.""" + if w <= box and h <= box: + return w, h + if w >= h: + return box, max(1, h * box // w) + return max(1, w * box // h), box + + +def fail(code, error, detail=""): + print(json.dumps({"error": error, "detail": detail[:200]})) + sys.stdout.flush() + os._exit(code) + + +def main(): + src, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) + limit = MEMORY_LIMIT_MB * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + resource.setrlimit(resource.RLIMIT_CPU, (CPU_LIMIT_SECONDS, CPU_LIMIT_SECONDS + 1)) + + import warnings + from PIL import Image, ImageOps + # Больше max_pixels — ошибка, а не предупреждение (по умолчанию Pillow только warn до 2x). + Image.MAX_IMAGE_PIXELS = max_pixels + warnings.simplefilter("error", Image.DecompressionBombWarning) + fmt = {"image/png": "PNG", "image/jpeg": "JPEG"}.get(content_type) + if fmt is None: + fail(EXIT_INVALID, "invalid_image", "unsupported content type") + try: + im = Image.open(src, formats=(fmt,)) + w0, h0 = im.size + if w0 * h0 > max_pixels: + fail(EXIT_TOO_LARGE, "image_too_large", "%dx%d" % (w0, h0)) + orientation = im.getexif().get(0x0112, 1) + swap = orientation in (5, 6, 7, 8) + ow, oh = (h0, w0) if swap else (w0, h0) # размеры ПОСЛЕ применения EXIF orientation + if fmt == "JPEG": + # draft: декодер JPEG сразу уменьшает в 2/4/8 раз, но не меньше 1280 по обеим + # сторонам — памяти нужно в 4–64 раза меньше, качество вариантов не страдает. + im.draft("RGB", (OPTIMIZED, OPTIMIZED)) + im.load() + if orientation in range(2, 9): + im = ImageOps.exif_transpose(im) + # tRNS бывает не только у палитры: у RGB и L это «прозрачный цвет» (аудит F06). + has_alpha = im.mode in ("RGBA", "LA", "PA") or "transparency" in im.info + work = im.convert("RGBA") if has_alpha else im.convert("RGB") + del im + # Промежуточная копия <= 1280: из неё делаются все варианты (экономия памяти и CPU). + mw, mh = fit_within(ow, oh, OPTIMIZED) + if work.size != (mw, mh): + work = work.resize((mw, mh), Image.LANCZOS, reducing_gap=3.0) + if has_alpha: + flat = Image.new("RGB", work.size, (255, 255, 255)) + flat.paste(work, mask=work.getchannel("A")) + else: + flat = work + work.info = {} # без метаданных источника (EXIF, XMP, ICC, comment...) + flat.info = {} + variants = [] + tw, th = fit_within(ow, oh, THUMB) + thumb = flat if flat.size == (tw, th) else flat.resize((tw, th), Image.LANCZOS) + path = os.path.join(out_dir, "thumbnail.jpg") + thumb.info = {} + thumb.save(path, "JPEG", quality=JPEG_QUALITY, optimize=True) + variants.append({"name": "thumbnail", "path": path, "content_type": "image/jpeg", "width": tw, "height": th}) + crop = ImageOps.fit(flat, (CROP, CROP), method=Image.LANCZOS, centering=(0.5, 0.5)) + crop.info = {} + path = os.path.join(out_dir, "crop.jpg") + crop.save(path, "JPEG", quality=JPEG_QUALITY, optimize=True) + variants.append({"name": "crop", "path": path, "content_type": "image/jpeg", "width": CROP, "height": CROP}) + path = os.path.join(out_dir, "optimized.webp") + work.save(path, "WEBP", quality=WEBP_QUALITY, method=4) + variants.append({"name": "optimized", "path": path, "content_type": "image/webp", "width": mw, "height": mh}) + except MemoryError: + fail(EXIT_RESOURCE, "resource_limit", "memory limit %d MiB" % MEMORY_LIMIT_MB) + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: + fail(EXIT_TOO_LARGE, "image_too_large", type(exc).__name__) + except Exception as exc: + # Битые/усечённые данные: OSError, SyntaxError, ValueError, struct.error... + fail(EXIT_INVALID, "invalid_image", type(exc).__name__) + print(json.dumps({"variants": variants, "source": [ow, oh]})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/vault.py b/src/vault.py new file mode 100644 index 0000000..7ae604a --- /dev/null +++ b/src/vault.py @@ -0,0 +1,195 @@ +"""Lab7: получение сервисных секретов из OpenBao. + +Аутентификация — метод cert (TLS client certificate сервиса, тот же leaf, что и для mTLS): + POST /v1/auth/cert/login {"name": BAO_AUTH_ROLE} -> client_token c policy только своего пути, + TTL 15 мин, max TTL 1 ч (задаётся ролью в OpenBao, см. scripts/bao-init.sh). +В процессе нет постоянного «секрета для получения секретов»: ни root token, ни BAO_TOKEN, ни +SecretID. Bootstrap trust = сертификат CA (/run/secrets/internal-ca.crt) + свой ключ (Docker secret). + +Затем: GET /v1//data/ (KV v2) — словарь значений {"POSTGRES_PASSWORD": ..., ...}. +Значения хранятся только в памяти процесса; в логи/телеметрию не пишутся (логируются лишь +имена полей и HTTP-статусы). + +Жизненный цикл токена: фоновый поток продлевает токен (renew-self) на половине TTL; если продлить +нельзя (истёк max TTL, токен отозван) — повторный login и перечитывание KV. Недоступный/sealed +OpenBao не останавливает уже работающий процесс: значения в памяти остаются, поток повторяет +попытки с паузой до 30 с. На старте без OpenBao процесс ждёт (BAO_STARTUP_WAIT_SECONDS) и падает, +не переходя на какие-либо значения по умолчанию. +""" +import json +import logging +import os +import threading +import time +import urllib.error +import urllib.request + +import tlsutil + +log = logging.getLogger("messenger.vault") + +BAO_ADDR = os.environ.get("BAO_ADDR", "").rstrip("/") +BAO_CACERT = os.environ.get("BAO_CACERT", tlsutil.CA_FILE) +BAO_AUTH_ROLE = os.environ.get("BAO_AUTH_ROLE", "") +BAO_KV_MOUNT = os.environ.get("BAO_KV_MOUNT", "kv") +BAO_KV_PATH = os.environ.get("BAO_KV_PATH", "") +STARTUP_WAIT = float(os.environ.get("BAO_STARTUP_WAIT_SECONDS", "300")) +TIMEOUT = float(os.environ.get("BAO_TIMEOUT_SECONDS", "5")) + + +class BaoError(Exception): + def __init__(self, status, message): + super().__init__("openbao %s: %s" % (status, message)) + self.status = status + + +def enabled(): + return bool(BAO_ADDR and BAO_KV_PATH) + + +def _context(): + import ssl + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=BAO_CACERT) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + if tlsutil.has_identity(): + ctx.load_cert_chain(tlsutil.CERT_FILE, tlsutil.KEY_FILE) + return ctx + + +def request(method, path, token=None, body=None, context=None): + """Один HTTP-вызов OpenBao API. Возвращает (status, json). Тело ответа не логируется.""" + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(BAO_ADDR + "/v1/" + path.lstrip("/"), data=data, method=method) + req.add_header("Content-Type", "application/json") + if token: + req.add_header("X-Vault-Token", token) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT, context=context or _context()) as resp: + raw = resp.read() + return resp.status, (json.loads(raw) if raw else {}) + except urllib.error.HTTPError as exc: + try: + payload = json.loads(exc.read() or b"{}") + except ValueError: + payload = {} + return exc.code, payload + + +class Client: + def __init__(self): + self._lock = threading.Lock() + self._token = None + self._ttl = 0 + self._renewable = False + self._values = None + self._thread = None + self.lease_info = {} + + # --- операции --- + def login(self): + status, payload = request("POST", "auth/cert/login", body={"name": BAO_AUTH_ROLE}) + if status != 200: + raise BaoError(status, "cert login failed (%s)" % ",".join(payload.get("errors", []))[:200]) + auth = payload["auth"] + self._token = auth["client_token"] + self._ttl = int(auth.get("lease_duration") or 0) + self._renewable = bool(auth.get("renewable")) + self.lease_info = {"policies": auth.get("token_policies", []), "ttl": self._ttl, + "renewable": self._renewable, "role": BAO_AUTH_ROLE} + log.info("openbao login ok role=%s policies=%s ttl=%ds", BAO_AUTH_ROLE, + ",".join(self.lease_info["policies"]), self._ttl, extra={"event": "vault.login"}) + + def read_kv(self): + path = "%s/data/%s" % (BAO_KV_MOUNT, BAO_KV_PATH) + status, payload = request("GET", path, token=self._token) + if status != 200: + raise BaoError(status, "read %s denied/failed" % path) + values = payload["data"]["data"] + version = payload["data"].get("metadata", {}).get("version") + log.info("openbao secret read path=%s version=%s fields=%s", path, version, ",".join(sorted(values)), + extra={"event": "vault.read"}) + return values + + def renew(self): + status, payload = request("POST", "auth/token/renew-self", token=self._token, body={}) + if status != 200: + raise BaoError(status, "renew failed") + auth = payload["auth"] + new_ttl = int(auth.get("lease_duration") or 0) + self._ttl = new_ttl + return new_ttl + + # --- жизненный цикл --- + def values(self): + with self._lock: + if self._values is None: + self._bootstrap() + return self._values + + def _bootstrap(self): + # Секреты нужны уже при импорте модулей (conninfo БД), до telemetry.setup(): включаем + # JSON-логи stdout заранее, чтобы ожидание OpenBao было видно структурным событием. + if not logging.getLogger().handlers: + import telemetry + telemetry.configure_logging() + deadline = time.monotonic() + STARTUP_WAIT + delay = 1.0 + while True: + try: + self.login() + self._values = self.read_kv() + break + except Exception as exc: # sealed (503), недоступен, ещё не инициализирован + if time.monotonic() > deadline: + raise SystemExit("openbao unavailable at startup: %s" % type(exc).__name__) + log.warning("openbao not ready (%s), retry in %.0fs", str(exc)[:120], delay, + extra={"event": "vault.unavailable"}) + time.sleep(delay) + delay = min(10.0, delay * 2) + self._thread = threading.Thread(target=self._maintain, name="vault-renew", daemon=True) + self._thread.start() + + def _maintain(self): + """Renew на половине TTL; не продлевается -> re-login + перечитать KV; ошибки -> backoff.""" + wait, backoff = max(5.0, self._ttl / 2.0), 2.0 + while True: + time.sleep(wait) + try: + ttl = self.renew() + if ttl < 60: # продление упёрлось в max TTL — пора новый login + raise BaoError(0, "max ttl reached") + log.info("openbao token renewed ttl=%ds", ttl, extra={"event": "vault.renewed"}) + wait, backoff = max(5.0, ttl / 2.0), 2.0 + continue + except Exception as exc: + reason = str(exc)[:80] + try: + self.login() + fresh = self.read_kv() + with self._lock: + self._values = fresh + wait, backoff = max(5.0, self._ttl / 2.0), 2.0 + except Exception as exc2: + wait, backoff = backoff, min(30.0, backoff * 2) + log.warning("openbao renew/login failed (%s / %s), keeping cached secrets, retry in %.0fs", + reason, str(exc2)[:80], wait, extra={"event": "vault.renew_failed"}) + + +CLIENT = Client() + + +def secret(name, required=True): + """Значение секрета: NAME_FILE (Docker secret, только инфраструктурные job/тесты), иначе + поле NAME из KV OpenBao (сервисы), иначе переменная окружения NAME (локальная разработка).""" + path = os.environ.get(name + "_FILE") + if path: + with open(path, encoding="utf-8") as f: + return f.read().strip() + if enabled(): + values = CLIENT.values() + if name in values: + return str(values[name]) + if not required: + return "" + raise SystemExit("secret %s is not present in openbao path %s" % (name, BAO_KV_PATH)) + return os.environ.get(name, "") diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..250350a --- /dev/null +++ b/src/worker.py @@ -0,0 +1,622 @@ +#!/usr/bin/env python3 +"""Worker обработки вложений (compose-сервис `worker`, отдельный процесс от API). + +WORKER_CONCURRENCY потоков-потребителей; у каждого своё соединение AMQP, prefetch=1 +(не больше одного неподтверждённого сообщения на поток) и своё соединение PostgreSQL. +Итого одновременно в работе не больше WORKER_CONCURRENCY заданий на контейнер. + +Обработка одного сообщения (at-least-once, ack только после устойчивого эффекта): + 1. разбор по jobs.schema.json; мусор -> basic.reject(requeue=False) -> DLQ; + 2. claim в транзакции (src/jobs.py): по строке jobs решаем, нужна ли работа. ready/failed, + чужая живая аренда, ранний дубль -> ack без работы. Иначе processing, attempt+1, + lease_token (fencing) и lease_until, COMMIT; + 3. пока идёт работа, отдельный поток продлевает аренду (heartbeat) не дольше + JOB_TIMEOUT_SECONDS: зависший процесс аренду теряет, долгий, но живой — нет; + 4. staging -> временный файл (лимит байт = size_bytes из БД, проверка SHA-256); + 5. transform.py в подпроцессе (RLIMIT_AS, RLIMIT_CPU, таймаут) -> thumbnail/crop/optimized; + 6. PUT оригинала и вариантов по ДЕТЕРМИНИРОВАННЫМ ключам attachments//v/: + повтор перезаписывает те же объекты, второго набора не появляется; + 7. complete: processing -> ready + attachment_variants одной транзакцией и только со своим + lease_token; иначе (аренду забрали) — ничего не пишем; + 8. удаление staging (после COMMIT), затем ack. Крах до ack -> redelivery -> claim видит + ready -> no-op ack. +Ошибки: постоянные -> failed(error_code); временные -> queued с backoff+jitter через outbox +(relay опубликует в срок) или failed(retry_exhausted). Нет доступа к БД -> сообщение +возвращается в очередь (nack) после паузы; если так и не записалось — reaper/redrive. + +Учебные отказы (WORKER_FAULT_INJECTION=1): filename вида fault-.png задаёт точку: +crash-after-claim, crash-after-s3-put, crash-after-commit (os._exit один раз на job), +transient- (временная ошибка в первых N попытках), transient-always (до replay), +bug (непредусмотренное исключение в 1-й попытке), +slow (пауза FAULT_SLOW_SECONDS после claim, аренда продлевается). + +Lab6 (src/telemetry.py): + * каждое сообщение — CONSUMER span «attachments.process.v1 process», родитель — контекст + из AMQP headers (traceparent от relay «send»), запасной вариант — поля traceparent/tracestate + тела (jobs.schema.json). Внутри: db jobs.claim, worker.fetch_staging (s3 GetObject), + worker.transform (подпроцесс), s3 PutObject x4, db jobs.complete, s3 DeleteObject staging; + * метрики: messenger.jobs.processed{messenger.job.type, messenger.job.outcome, error.type}, + messenger.job.duration{type, outcome} (s, от claim до итога), messenger.job.queue_wait (s, от + next_attempt_at до claim), messenger.attachment.time_to_ready{outcome} (s, от durable acceptance + = jobs.created_at до COMMIT ready/failed), messenger.worker.busy / messenger.worker.capacity + (мгновенно) и messenger.worker.busy_time (s, счётчик: rate/capacity = точная загрузка). +""" +import concurrent.futures +import hashlib +import json +import logging +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time + +import psycopg +from psycopg.rows import dict_row + +import common +import jobs +import objstore +import telemetry +from objstore import ObjectMissing, StorageUnavailable +from telemetry import SpanKind, Status, StatusCode + +log = logging.getLogger("worker") +CONCURRENCY = common.env_int("WORKER_CONCURRENCY", 2) +PREFETCH = common.env_int("WORKER_PREFETCH", 1) +LEASE_SECONDS = common.env_int("JOB_LEASE_SECONDS", 30) +JOB_TIMEOUT = common.env_float("JOB_TIMEOUT_SECONDS", 60) +RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2) +RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60) +MAX_UPLOAD_BYTES = common.env_int("MAX_UPLOAD_BYTES", 10485760) +MAX_IMAGE_PIXELS = common.env_int("MAX_IMAGE_PIXELS", 40000000) +REQUEUE_PAUSE = common.env_float("WORKER_REQUEUE_PAUSE_SECONDS", 2) +WORK_DIR = os.environ.get("WORKER_TMP_DIR") or None +if WORK_DIR: # lab7: tmpfs поверх read-only rootfs + os.makedirs(WORK_DIR, mode=0o700, exist_ok=True) +FAULT_INJECTION = os.environ.get("WORKER_FAULT_INJECTION", "0") == "1" +FAULT_SLOW_SECONDS = common.env_float("FAULT_SLOW_SECONDS", 20) +TRANSFORM = os.path.join(os.path.dirname(os.path.abspath(__file__)), "transform.py") +HEARTBEAT = "/tmp/worker.heartbeat" +EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"} +TRANSFORM_ERRORS = {3: "invalid_image", 4: "image_too_large", 5: "resource_limit"} + +OBJ = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments")) +STOP = threading.Event() + +# --- Lab6: метрики worker --- +_m = telemetry.meter +JOBS_PROCESSED = _m.create_counter("messenger.jobs.processed", unit="{job}", + description="Job messages handled by the worker, by outcome") +JOB_DURATION = _m.create_histogram("messenger.job.duration", unit="s", + description="Processing time of a claimed job attempt (claim -> outcome)") +QUEUE_WAIT = _m.create_histogram("messenger.job.queue_wait", unit="s", + description="Time a due job waited before a worker claimed it") +TIME_TO_READY = _m.create_histogram("messenger.attachment.time_to_ready", unit="s", + description="From durable acceptance (enqueue COMMIT) to ready/failed COMMIT") +BUSY_TIME = _m.create_counter("messenger.worker.busy_time", unit="s", + description="Cumulative consumer time spent handling messages (utilization = rate / capacity)") +_BUSY = [0] +_BUSY_LOCK = threading.Lock() + + +def _busy(delta): + with _BUSY_LOCK: + _BUSY[0] += delta + + +_m.create_observable_gauge("messenger.worker.busy", [lambda o: [telemetry.metrics.Observation(_BUSY[0])]], + unit="{consumer}", description="Consumers currently processing a job") +_m.create_observable_gauge("messenger.worker.capacity", [lambda o: [telemetry.metrics.Observation(CONCURRENCY)]], + unit="{consumer}", description="Configured consumer threads (WORKER_CONCURRENCY)") +JOB_ATTRS = {"messenger.job.type": jobs.JOB_TYPE} + + +class JobError(Exception): + def __init__(self, code, detail=""): + super().__init__(code) + self.code = code + self.detail = detail + + +class LeaseLost(Exception): + pass + + +class Lease: + """Продление аренды отдельным соединением раз в LEASE_SECONDS/3, но не дольше единого + срока задания (JOB_TIMEOUT_SECONDS от claim): после него аренда больше не продлевается + и reaper вернёт задание в очередь, даже если процесс жив, но завис. + + Аудит F13: срок — общий бюджет на чтение staging -> transform -> запись результатов. + check() перед каждым шагом с побочным эффектом прерывает задание, если срок вышел + (processing_timeout, обычный retry), и больше не продлевает аренду после срока.""" + + def __init__(self, conninfo, job, deadline): + self.conninfo = conninfo + self.job = job + self.lost = False + self.deadline = deadline + self._stop = threading.Event() + self._thread = threading.Thread(target=self._loop, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc): + self._stop.set() + self._thread.join(timeout=5) + + def left(self): + return self.deadline - time.monotonic() + + def check(self, conn=None): + """Перед каждым побочным эффектом: флаг фонового heartbeat И синхронная проверка + (продление) аренды в БД. Иначе замороженный процесс после разморозки успел бы + сделать шаг до того, как фоновый поток заметит потерю аренды.""" + if self.lost: + raise LeaseLost() + if self.left() <= 0: + raise JobError("processing_timeout", "job deadline %.0fs exceeded" % JOB_TIMEOUT) + if conn is not None: + with conn.transaction(): + ok = jobs.heartbeat(conn, self.job["job_id"], self.job["lease_token"], LEASE_SECONDS) + if not ok: + self.lost = True + raise LeaseLost() + + def _loop(self): + conn = None + while not self._stop.wait(max(1.0, LEASE_SECONDS / 3)): + if time.monotonic() > self.deadline: + log.warning("job %s exceeded hard deadline, lease is no longer renewed", self.job["job_id"], + extra={"event": "job.deadline_exceeded", "job_id": str(self.job["job_id"])}) + return + try: + if conn is None or conn.closed: + conn = psycopg.connect(self.conninfo, autocommit=True) + if not jobs.heartbeat(conn, self.job["job_id"], self.job["lease_token"], LEASE_SECONDS): + self.lost = True + log.warning("lease lost for job %s", self.job["job_id"], + extra={"event": "job.lease_lost", "job_id": str(self.job["job_id"])}) + return + except psycopg.Error as exc: + log.warning("lease heartbeat failed (%s)", type(exc).__name__, + extra={"event": "job.heartbeat_failed", "error.type": type(exc).__name__}) + conn = None + if conn is not None: + conn.close() + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +class Consumer(threading.Thread): + def __init__(self, index): + super().__init__(name="consumer-%d" % index, daemon=True) + self.conninfo = common.database_conninfo() + self.db = None + self.last_beat = time.monotonic() + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + + # ------------------------------------------------------------------ AMQP + def run(self): + while not STOP.is_set(): + conn = common.connect_broker(log, STOP) + if conn is None: + return + try: + ch = conn.channel() + ch.basic_qos(prefetch_count=PREFETCH) + log.info("%s consuming %s owner=%s", self.name, common.QUEUE, common.OWNER, + extra={"event": "worker.consuming"}) + for method, props, body in ch.consume(common.QUEUE, inactivity_timeout=1): + self.last_beat = time.monotonic() + if STOP.is_set(): + break + if method is None: + continue + outcome = self.run_job(conn, body, getattr(props, "headers", None) or {}, + bool(getattr(method, "redelivered", False))) + if outcome == "ack": + ch.basic_ack(method.delivery_tag) + elif outcome == "reject": + ch.basic_reject(method.delivery_tag, requeue=False) # -> DLX -> DLQ + else: + self.pause(conn, REQUEUE_PAUSE) + ch.basic_nack(method.delivery_tag, requeue=True) + ch.cancel() + conn.close() + except Exception as exc: + log.warning("%s broker loop error (%s), reconnecting", self.name, type(exc).__name__, + extra={"event": "broker.loop_error", "error.type": type(exc).__name__}) + try: + conn.close() + except Exception: + pass + STOP.wait(1) + + def pause(self, conn, seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + conn.process_data_events(time_limit=min(0.5, end - time.monotonic())) + self.last_beat = time.monotonic() + + def run_job(self, conn, body, headers=None, redelivered=False): + """Работа в отдельном потоке; этот поток обслуживает heartbeat AMQP-соединения.""" + future = self.pool.submit(self.traced_message, body, headers or {}, redelivered) + # Lab6 (найдено по метрикам инцидента 1): process_data_events(time_limit=0.5) ждал полные + # 0.5 с и после конца задания — потребитель брал не больше 2 заданий/с (utilization 41 %). + # Пустой threadsafe-callback по завершении будит цикл AMQP сразу. + future.add_done_callback(lambda _f: conn.add_callback_threadsafe(lambda: None)) + while not future.done(): + conn.process_data_events(time_limit=0.5) + self.last_beat = time.monotonic() + try: + return future.result() + except Exception: + log.exception("unexpected worker error", extra={"event": "worker.unexpected_error"}) + return "requeue" + + # ------------------------------------------------------------------ DB + def database(self): + if self.db is None or self.db.closed: + self.db = psycopg.connect(self.conninfo, row_factory=dict_row) + return self.db + + def drop_db(self): + try: + if self.db is not None: + self.db.close() + except Exception: + pass + self.db = None + + # ------------------------------------------------------------------ job + @staticmethod + def _parent_context(body, headers): + carrier = {} + for key in ("traceparent", "tracestate"): + value = headers.get(key) + if isinstance(value, bytes): + value = value.decode("ascii", "replace") + if isinstance(value, str) and value: + carrier[key] = value + if "traceparent" not in carrier: + try: + raw = json.loads(body.decode("utf-8")) + for key in ("traceparent", "tracestate"): + if isinstance(raw.get(key), str): + carrier[key] = raw[key] + except Exception: + pass + return telemetry.extract(carrier) + + def traced_message(self, body, headers, redelivered): + """CONSUMER span вокруг обработки + метрики исхода. res['outcome'] заполняет обработка.""" + res = {"outcome": "unknown", "error": None} + span = telemetry.tracer.start_span( + "%s process" % common.QUEUE, kind=SpanKind.CONSUMER, context=self._parent_context(body, headers), + attributes={"messaging.system": "rabbitmq", "messaging.operation.type": "process", + "messaging.operation.name": "process", "messaging.destination.name": common.QUEUE, + "messaging.rabbitmq.destination.routing_key": common.ROUTING_KEY, + "messaging.message.body.size": len(body), "messaging.message.redelivered": redelivered, + "messenger.worker.owner": common.OWNER}) + _busy(1) + busy_started = time.perf_counter() + try: + with telemetry.trace.use_span(span, end_on_exit=False): + action = self.process_message(body, res, span) + return action + except BaseException as exc: + res["outcome"], res["error"] = "crashed", type(exc).__name__ + span.record_exception(exc) + raise + finally: + _busy(-1) + BUSY_TIME.add(time.perf_counter() - busy_started) + span.set_attribute("messenger.job.outcome", res["outcome"]) + attrs = dict(JOB_ATTRS, **{"messenger.job.outcome": res["outcome"]}) + if res["error"]: + attrs["error.type"] = res["error"] + span.set_attribute("error.type", res["error"]) + if res["outcome"] in ("failed", "rejected", "requeued", "crashed"): + span.set_status(Status(StatusCode.ERROR, res["error"] or res["outcome"])) + JOBS_PROCESSED.add(1, attrs) + span.end() + + def process_message(self, body, res=None, span=None): + res = res if res is not None else {} + try: + msg = jobs.parse_message(body) + except jobs.InvalidMessage as exc: + log.warning("invalid message rejected to DLQ (%s)", exc, + extra={"event": "job.rejected", "reason": str(exc)}) + res.update(outcome="rejected", error="invalid_message") + return "reject" + if span is not None: + span.set_attribute("messaging.message.id", "%s:%s" % (msg["job_id"], msg["attempt"])) + span.set_attribute("messenger.job.id", msg["job_id"]) + span.set_attribute("messenger.attachment.id", msg["attachment_id"]) + span.set_attribute("messenger.job.message_attempt", msg["attempt"]) + try: + db = self.database() + with db.transaction(): + action, job = jobs.claim(db, msg, common.OWNER, LEASE_SECONDS) + except psycopg.Error as exc: + log.warning("claim failed, message requeued (%s)", type(exc).__name__, + extra={"event": "job.requeued", "error.type": type(exc).__name__, "job_id": msg["job_id"]}) + self.drop_db() + res.update(outcome="requeued", error="db_unavailable") + return "requeue" + if action == "reject": + log.warning("message for job %s rejected to DLQ (%s)", msg["job_id"], job, + extra={"event": "job.rejected", "job_id": msg["job_id"], "reason": str(job)}) + res.update(outcome="rejected", error=str(job)) + return "reject" + if action == "ack": + res.update(outcome="duplicate") + return "ack" + QUEUE_WAIT.record(float(job.get("eligible_wait_s") or 0.0), JOB_ATTRS) + if span is not None: + span.set_attribute("messenger.job.attempt", job["attempt"]) + span.set_attribute("messenger.job.queue_wait_s", round(float(job.get("eligible_wait_s") or 0.0), 3)) + log.info("job %s attempt %d claimed (attachment %s)", job["job_id"], job["attempt"], job["attachment_id"], + extra={"event": "job.claimed", "job_id": str(job["job_id"]), + "attachment_id": str(job["attachment_id"]), "attempt": job["attempt"]}) + started = time.perf_counter() + try: + return self.execute(db, job, res) + except psycopg.Error as exc: + # Результат не записан; аренда истечёт, reaper вернёт задание. Сообщение — назад. + log.warning("database error during job %s (%s)", job["job_id"], type(exc).__name__, + extra={"event": "job.requeued", "job_id": str(job["job_id"]), "error.type": type(exc).__name__}) + self.drop_db() + res.update(outcome="requeued", error="db_unavailable") + return "requeue" + finally: + JOB_DURATION.record(time.perf_counter() - started, + dict(JOB_ATTRS, **{"messenger.job.outcome": res.get("outcome", "unknown")})) + + def fault_point(self, job): + if not FAULT_INJECTION: + return None + name = job["filename"] or "" + if not name.startswith("fault-"): + return None + return name[len("fault-"):].rsplit(".", 1)[0] + + def crash_once(self, db, job, point, wanted): + if point != wanted: + return + with db.transaction(): + seen = db.execute( + """SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected' + AND detail->>'point' = %s""", (job["job_id"], point)).fetchone() + if seen: + return + jobs.add_event(db, job["job_id"], "fault_injected", job["attempt"], common.OWNER, {"point": point}) + log.warning("FAULT INJECTION %s job=%s: process exits now", point, job["job_id"], + extra={"event": "fault.injected", "fault.point": point, "job_id": str(job["job_id"])}) + telemetry.shutdown(1000) # успеть отправить spans/logs до «краха» (учебный сценарий) + os._exit(86) + + def execute(self, db, job, res=None): + res = res if res is not None else {} + point = self.fault_point(job) + started = time.monotonic() + with Lease(self.conninfo, job, started + JOB_TIMEOUT) as lease: + try: + self.crash_once(db, job, point, "crash-after-claim") + if point == "slow" and job["attempt"] == 1: + log.warning("FAULT INJECTION slow job=%s sleeping %.0fs", job["job_id"], FAULT_SLOW_SECONDS) + time.sleep(FAULT_SLOW_SECONDS) + if point == "bug" and job["attempt"] == 1: + raise RuntimeError("injected programming error") + if point and point.startswith("transient-"): + n = point.split("-", 1)[1] + if (n == "always" and job["replay_count"] == 0) or (n.isdigit() and job["attempt"] <= int(n)): + raise JobError("storage_unavailable", "injected transient error") + lease.check(db) + variants = self.do_work(db, job, lease) + self.crash_once(db, job, point, "crash-after-s3-put") + lease.check(db) + except LeaseLost: + with db.transaction(): + jobs.add_event(db, job["job_id"], "aborted_lease_lost", job["attempt"], common.OWNER) + res.update(outcome="lease_lost") + log.warning("job %s attempt %d: lease lost, work aborted", job["job_id"], job["attempt"], + extra={"event": "job.lease_lost", "job_id": str(job["job_id"]), "attempt": job["attempt"]}) + return "ack" + except JobError as err: + with db.transaction(): + result, delay = jobs.fail_or_retry(db, job, err.code, err.detail, common.OWNER, + RETRY_BASE, RETRY_MAX) + res.update(outcome=result, error=err.code) + self._record_terminal(job, result) + log.warning("job %s attempt %d: %s (%s) -> %s%s", job["job_id"], job["attempt"], err.code, + err.detail, result, " in %.1fs" % delay if delay else "", + extra={"event": "job.retry_scheduled" if result == "retry" else "job." + result, + "job_id": str(job["job_id"]), "attempt": job["attempt"], + "error_code": err.code, "retry_delay_s": round(delay or 0, 3)}) + return "ack" + except psycopg.Error: + raise # нет БД: сообщение назад в очередь (process_message) + except Exception as exc: + # Непредусмотренная ошибка (баг, неожиданный вывод transform): временная ошибка + # с обычным retry/backoff, а не «висящее» processing до истечения аренды. + log.exception("job %s attempt %d: unexpected error", job["job_id"], job["attempt"], + extra={"event": "job.unexpected_error", "job_id": str(job["job_id"]), + "attempt": job["attempt"]}) + with db.transaction(): + result, delay = jobs.fail_or_retry(db, job, "processing_failed", type(exc).__name__, + common.OWNER, RETRY_BASE, RETRY_MAX) + res.update(outcome=result, error="processing_failed") + self._record_terminal(job, result) + return "ack" + with db.transaction(): + result = jobs.complete(db, job, variants, common.OWNER) + res.update(outcome=result) + self._record_terminal(job, result) + self.crash_once(db, job, point, "crash-after-commit") + log.info("job %s attempt %d -> %s in %.0fms", job["job_id"], job["attempt"], result, + (time.monotonic() - started) * 1000, + extra={"event": "job.completed" if result == "ready" else "job." + result, + "job_id": str(job["job_id"]), "attachment_id": str(job["attachment_id"]), + "attempt": job["attempt"], "duration_ms": round((time.monotonic() - started) * 1000, 1), + "time_to_ready_s": round(job.get("time_to_ready_s") or 0.0, 3)}) + if result == "ready": + self.cleanup_staging(db, job) + return "ack" + + @staticmethod + def _record_terminal(job, result): + if result == "ready" and job.get("time_to_ready_s") is not None: + TIME_TO_READY.record(job["time_to_ready_s"], dict(JOB_ATTRS, outcome="ready")) + elif result == "failed" and job.get("time_to_terminal_s") is not None: + TIME_TO_READY.record(job["time_to_terminal_s"], dict(JOB_ATTRS, outcome="failed")) + + def cleanup_staging(self, db, job): + """Staging удаляется только после COMMIT ready. Не вышло — доберёт janitor.""" + key = job["staging_key"] + if not key: + return + try: + OBJ.delete(key) + except StorageUnavailable: + log.warning("staging cleanup postponed for %s", job["attachment_id"], + extra={"event": "staging.cleanup_postponed", "attachment_id": str(job["attachment_id"])}) + return + with db.transaction(): + db.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'", + (job["attachment_id"],)) + jobs.add_event(db, job["job_id"], "staging_deleted", job["attempt"], common.OWNER) + + def do_work(self, db, job, lease): + tmpdir = tempfile.mkdtemp(prefix="job-", dir=WORK_DIR) + try: + src = os.path.join(tmpdir, "source") + with telemetry.child_span("worker.fetch_staging", attributes={"messenger.size_bytes": job["size_bytes"]}): + self.fetch_staging(job, src, lease) + lease.check(db) + with telemetry.child_span("worker.transform", attributes={ + "messenger.content_type": job["content_type"], "messenger.pipeline_version": job["pipeline_version"], + "messenger.image.pixels": (job["width"] or 0) * (job["height"] or 0)}): + produced = self.transform(job, src, tmpdir, lease) + lease.check(db) # последняя проверка перед записью в конечный prefix + ver = job["pipeline_version"] + base = "attachments/%s/v%d/" % (job["attachment_id"], ver) + if job["object_key"] != base + "original": + raise JobError("staging_corrupted", "unexpected object key") + variants = [] + for v in produced: + path = v["path"] + variants.append({"name": v["name"], "key": base + "%s.%s" % (v["name"], EXT[v["content_type"]]), + "content_type": v["content_type"], "width": v["width"], "height": v["height"], + "size_bytes": os.path.getsize(path), "sha256": sha256_file(path), "path": path}) + try: + # Сначала все объекты, потом COMMIT в БД: ready никогда не ссылается на + # отсутствующий объект. Ключи детерминированы — повтор перезаписывает их же. + with open(src, "rb") as f: + OBJ.put(job["object_key"], f, job["size_bytes"], job["content_type"]) + for v in variants: + with open(v["path"], "rb") as f: + OBJ.put(v["key"], f, v["size_bytes"], v["content_type"]) + except StorageUnavailable: + raise JobError("storage_unavailable", "put result") + return variants + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def fetch_staging(self, job, path, lease): + key = job["staging_key"] + if not key: + raise JobError("staging_missing", "staging already removed") + limit = min(job["size_bytes"], MAX_UPLOAD_BYTES) + try: + resp = OBJ.open(key) + except ObjectMissing: + raise JobError("staging_missing", "no staging object") + except StorageUnavailable: + raise JobError("storage_unavailable", "get staging") + h = hashlib.sha256() + size = 0 + try: + with open(path, "wb") as f: + for chunk in resp.stream(objstore.STREAM_CHUNK): + if lease.left() <= 0: # медленный S3 не растягивает задание сверх срока + raise JobError("processing_timeout", "staging read exceeded job deadline") + size += len(chunk) + if size > limit: + raise JobError("staging_corrupted", "staging larger than metadata") + h.update(chunk) + f.write(chunk) + except JobError: + raise + except Exception as exc: + raise JobError("storage_unavailable", "read staging: %s" % type(exc).__name__) + finally: + resp.close() + resp.release_conn() + if size != job["size_bytes"] or h.hexdigest() != job["sha256"]: + raise JobError("staging_corrupted", "size/sha256 differ from accepted upload") + + def transform(self, job, src, tmpdir, lease): + out = os.path.join(tmpdir, "out") + os.mkdir(out) + left = lease.left() + if left <= 0: # бюджет исчерпан — transform даже не запускаем (аудит F13) + raise JobError("processing_timeout", "no time left for transform") + try: + proc = subprocess.run( + [sys.executable, TRANSFORM, src, out, job["content_type"], str(MAX_IMAGE_PIXELS)], + capture_output=True, timeout=left, check=False) + except subprocess.TimeoutExpired: + raise JobError("processing_timeout", "transform exceeded %.0fs" % left) + if proc.returncode < 0: + raise JobError("resource_limit", "transform killed by signal %d" % -proc.returncode) + if proc.returncode in TRANSFORM_ERRORS: + raise JobError(TRANSFORM_ERRORS[proc.returncode], proc.stdout.decode(errors="replace")[:200]) + if proc.returncode != 0: + raise JobError("processing_failed", "transform exit %d" % proc.returncode) + result = json.loads(proc.stdout) + names = sorted(v["name"] for v in result["variants"]) + if names != sorted(jobs.VARIANT_NAMES): + raise JobError("processing_failed", "unexpected variants") + return result["variants"] + + +def main(): + telemetry.setup("messenger-worker") + signal.signal(signal.SIGTERM, lambda *_: STOP.set()) + signal.signal(signal.SIGINT, lambda *_: STOP.set()) + log.info("worker starting owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%.0fs fault_injection=%s", + common.OWNER, CONCURRENCY, PREFETCH, LEASE_SECONDS, JOB_TIMEOUT, FAULT_INJECTION, + extra={"event": "worker.started"}) + consumers = [Consumer(i) for i in range(CONCURRENCY)] + for c in consumers: + c.start() + while not STOP.is_set(): + # healthcheck: файл обновляется, только пока все потребители живы и не зависли. + if all(c.is_alive() and time.monotonic() - c.last_beat < 30 for c in consumers): + common.touch(HEARTBEAT) + STOP.wait(2) + log.info("SIGTERM: finishing in-flight jobs", extra={"event": "worker.draining"}) + for c in consumers: + c.join(timeout=JOB_TIMEOUT + 30) + log.info("worker stopped", extra={"event": "worker.stopped"}) + telemetry.shutdown() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/argon2_bench.py b/student-tests/argon2_bench.py new file mode 100644 index 0000000..f80b36d --- /dev/null +++ b/student-tests/argon2_bench.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Lab7: замер Argon2id ВНУТРИ API-контейнера (те же лимиты cgroup: 0.5 CPU, 640 MiB). + + docker compose exec -T api python - < student-tests/argon2_bench.py > evidence/argon2-bench.json + +Для каждого профиля: N хешей подряд и N проверок, p50/p95/max в мс, пиковый RSS-прирост процесса +(ru_maxrss) — оценка памяти одного хеша. Плюс 2 параллельных хеша (PASSWORD_HASH_CONCURRENCY=2): +сколько входов/с выдержит реплика при насыщении CPU. Пароли синтетические. +""" +import json +import os +import platform +import resource +import statistics +import threading +import time + +from argon2 import PasswordHasher, Type + +PROFILES = [ + ("owasp-min m=19MiB t=2 p=1 (выбран)", 19456, 2, 1), + ("owasp-alt m=46MiB t=1 p=1", 47104, 1, 1), + ("m=19MiB t=3 p=1", 19456, 3, 1), + ("m=64MiB t=3 p=1", 65536, 3, 1), + ("m=19MiB t=2 p=2", 19456, 2, 2), +] +N = int(os.environ.get("BENCH_N", "15")) + + +def pct(xs, q): + xs = sorted(xs) + return xs[min(len(xs) - 1, int(round(q * (len(xs) - 1))))] + + +def cpu_limit(): + try: + quota, period = open("/sys/fs/cgroup/cpu.max").read().split() + return None if quota == "max" else int(quota) / int(period) + except OSError: + return None + + +def mem_limit(): + try: + v = open("/sys/fs/cgroup/memory.max").read().strip() + return None if v == "max" else int(v) >> 20 + except OSError: + return None + + +out = {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "python": platform.python_version(), + "machine": platform.machine(), "cgroup_cpu_limit": cpu_limit(), "cgroup_memory_limit_mib": mem_limit(), + "n_per_profile": N, "profiles": []} +for name, m, t, p in PROFILES: + ph = PasswordHasher(time_cost=t, memory_cost=m, parallelism=p, hash_len=32, salt_len=16, type=Type.ID) + rss0 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + hashes, h_ms, v_ms = [], [], [] + for i in range(N): + pw = "bench-password-%04d-xyz" % i + s = time.perf_counter() + hashes.append(ph.hash(pw)) + h_ms.append((time.perf_counter() - s) * 1000) + s = time.perf_counter() + ph.verify(hashes[-1], pw) + v_ms.append((time.perf_counter() - s) * 1000) + rss1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # 2 параллельных хеша (как PASSWORD_HASH_CONCURRENCY=2), 2*N операций + done = [] + lock = threading.Lock() + + def worker(k): + for i in range(N): + ph.hash("par-%d-%d" % (k, i)) + with lock: + done.append(1) + s = time.perf_counter() + ths = [threading.Thread(target=worker, args=(k,)) for k in range(2)] + [x.start() for x in ths] + [x.join() for x in ths] + wall = time.perf_counter() - s + out["profiles"].append({ + "name": name, "memory_kib": m, "iterations": t, "parallelism": p, + "hash_ms": {"p50": round(statistics.median(h_ms), 1), "p95": round(pct(h_ms, 0.95), 1), "max": round(max(h_ms), 1)}, + "verify_ms": {"p50": round(statistics.median(v_ms), 1), "p95": round(pct(v_ms, 0.95), 1)}, + "max_rss_growth_mib": round((rss1 - rss0) / 1024, 1), + "concurrency2_hashes_per_s": round(len(done) / wall, 1), + "sample_prefix": hashes[0].split("$")[1:4], + }) +print(json.dumps(out, ensure_ascii=False, indent=1)) diff --git a/student-tests/broker_permissions.py b/student-tests/broker_permissions.py new file mode 100644 index 0000000..d2343a2 --- /dev/null +++ b/student-tests/broker_permissions.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Права пользователей RabbitMQ (минимальные привилегии). Хост-оркестратор: + + python3 student-tests/broker_permissions.py --output evidence/broker-permissions.json + +Внутри контейнеров relay, worker и ops (каждый — со СВОИМ BROKER_URL_FILE) выполняется +проба (--probe): publish в exchange attachments (с mandatory и неизвестным routing key — +сообщение никуда не попадает, но право write проверяется), basic.get из рабочей очереди и +из DLQ (право read), queue.declare (право configure). Ожидаемая матрица: + relay — publish; worker — read рабочей очереди; ops — publish + read DLQ; configure — никто. +Также: пользователя guest нет; порты брокера на хост не опубликованы. +""" +import argparse +import json +import subprocess +import sys +from pathlib import Path + +EXPECT = { + "relay": {"publish": True, "get_work_queue": False, "get_dlq": False, "declare_queue": False}, + "worker": {"publish": False, "get_work_queue": True, "get_dlq": False, "declare_queue": False}, + "ops": {"publish": True, "get_work_queue": False, "get_dlq": True, "declare_queue": False}, +} + + +def probe(): + sys.path.insert(0, "/app/src") + import pika + import common + out = {} + conn = pika.BlockingConnection(common.broker_parameters()) + + def attempt(name, fn): + ch = conn.channel() + try: + fn(ch) + out[name] = True + except pika.exceptions.UnroutableError: + out[name] = True # write разрешён, просто нет binding + except pika.exceptions.ChannelClosedByBroker as exc: + out[name] = False if exc.reply_code == 403 else "error %d" % exc.reply_code + finally: + if ch.is_open: + ch.close() + + def publish(ch): + ch.confirm_delivery() + ch.basic_publish(common.EXCHANGE, "permission.check.unbound", b"{}", mandatory=True) + + def get(queue): + def fn(ch): + m, _p, _b = ch.basic_get(queue, auto_ack=False) + if m is not None: + ch.basic_nack(m.delivery_tag, requeue=True) + return fn + + attempt("publish", publish) + attempt("get_work_queue", get(common.QUEUE)) + attempt("get_dlq", get(common.DLQ)) + attempt("declare_queue", lambda ch: ch.queue_declare("evil-queue", durable=True)) + conn.close() + print(json.dumps(out)) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--probe", action="store_true") + p.add_argument("--output") + args = p.parse_args() + if args.probe: + return probe() + root = Path(__file__).resolve().parents[1] + script = (root / "student-tests/broker_permissions.py").read_text() + results, ok = {}, True + runners = {"relay": ["docker", "compose", "exec", "-T", "relay", "python", "-", "--probe"], + "worker": ["docker", "compose", "exec", "-T", "worker", "python", "-", "--probe"], + "ops": ["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "--entrypoint", "python", + "ops", "-", "--probe"]} + for user, cmd in runners.items(): + r = subprocess.run(cmd, input=script, capture_output=True, text=True, cwd=root, timeout=120) + got = json.loads(r.stdout.strip().splitlines()[-1]) + results[user] = {"got": got, "expected": EXPECT[user], "pass": got == EXPECT[user]} + ok &= got == EXPECT[user] + print("%s %-6s %s" % ("PASS" if got == EXPECT[user] else "FAIL", user, got)) + users = subprocess.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"], + capture_output=True, text=True, cwd=root).stdout.split() + ports = subprocess.run(["docker", "compose", "port", "broker", "5672"], capture_output=True, text=True, cwd=root) + results["guest_absent"] = "guest" not in users + results["broker_port_published_to_host"] = bool(ports.stdout.strip()) + ok &= results["guest_absent"] and not results["broker_port_published_to_host"] + print("guest absent:", results["guest_absent"], "| port 5672 on host:", results["broker_port_published_to_host"]) + if args.output: + Path(args.output).write_text(json.dumps(results, ensure_ascii=False, indent=1) + "\n") + print("PASS" if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/student-tests/burst.py b/student-tests/burst.py new file mode 100644 index 0000000..7aa1a1d --- /dev/null +++ b/student-tests/burst.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Уровень 4: ограниченная конкуренция/prefetch, очистка staging, память worker под всплеском. + + python3 student-tests/burst.py --count 24 --output evidence/burst.json + +N несжимаемых PNG 1500x1500 (~6.8 MB) загружаются 8 потоками. Во время обработки раз в +секунду снимается память контейнеров worker (docker stats). По job_events считаются: +максимальное число одновременно обрабатываемых заданий (claimed..ready), задержка очереди +(enqueued -> claimed) и время до ready. Ожидание: одновременно <= WORKER_REPLICAS x +WORKER_CONCURRENCY (prefetch=1 на потребителя), staging всех заданий удалён. +Иллюстрация на одной машине, не статистика. +""" +import argparse +import concurrent.futures +import datetime as dt +import json +import statistics +import subprocess +import threading +import time + +from lab5lib import Client, Log, check, compose, noise_png_fast, queues, sql, upload, wait_status + + +def ts(text): + return dt.datetime.fromisoformat(text.replace("Z", "+00:00")) + + +def mem_sampler(stop, samples): + while not stop.is_set(): + ids = compose("ps", "-q", "worker").stdout.split() + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}} {{.CPUPerc}}"] + + ids, capture_output=True, text=True).stdout + for line in out.splitlines(): + name, used, _slash, limit, cpu = line.split()[:5] + samples.append({"name": name, "used": used, "limit": limit, "cpu": cpu}) + stop.wait(1) + + +def mib(text): + num = float("".join(ch for ch in text if ch.isdigit() or ch == ".")) + return num / 1024 if text.endswith("KiB") else num * 1024 if text.endswith("GiB") else num + + +def pct(values, p): + values = sorted(values) + return round(values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))], 3) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--count", type=int, default=24) + p.add_argument("--output") + args = p.parse_args() + log = Log("burst of uploads: bounded worker concurrency") + c = Client(args.base_url, timeout=60) + alice, bob, _eve, chat = c.bootstrap() + consumers = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "-p", "messenger", "list_consumers", + "queue_name", "prefetch_count", "ack_required", "--formatter", "json").stdout + consumers = json.loads(consumers) + log.step("consumers", count=len(consumers), prefetch=sorted({x["prefetch_count"] for x in consumers})) + images = [noise_png_fast(1500, 1500) for _ in range(4)] + log.step("generated images", bytes=[len(x) for x in images]) + stop = threading.Event() + samples = [] + sampler = threading.Thread(target=mem_sampler, args=(stop, samples), daemon=True) + sampler.start() + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(8) as pool: + atts = list(pool.map(lambda i: upload(c, alice, chat, images[i % 4], "burst-%d.png" % i), + range(args.count))) + t_up = time.monotonic() - t0 + q = queues()["attachments.process.v1"] + log.step("all uploads accepted (202)", count=len(atts), seconds=round(t_up, 2), queue_right_after=q) + for a in atts: + st = wait_status(c, bob, a["id"], {"ready", "failed"}, timeout=180, interval=1) + check(st["status"] == "ready", "burst job failed") + total = time.monotonic() - t0 + stop.set() + sampler.join() + ids = ",".join("'%s'" % a["id"] for a in atts) + rows = sql("""SELECT j.attachment_id, e.event, to_char(e.at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') + FROM job_events e JOIN jobs j ON j.id = e.job_id + WHERE j.attachment_id IN (%s) AND e.event IN ('enqueued','claimed','ready') ORDER BY e.id""" % ids) + per = {} + for att_id, ev, at in rows: + per.setdefault(att_id, {})[ev] = ts(at) + intervals = [(v["claimed"], v["ready"]) for v in per.values()] + points = sorted([(a, 1) for a, _ in intervals] + [(b, -1) for _, b in intervals], key=lambda x: (x[0], x[1])) + cur = peak = 0 + for _, d in points: + cur += d + peak = max(peak, cur) + lag = [(v["claimed"] - v["enqueued"]).total_seconds() for v in per.values()] + to_ready = [(v["ready"] - v["enqueued"]).total_seconds() for v in per.values()] + work = [(v["ready"] - v["claimed"]).total_seconds() for v in per.values()] + staging_left = sql("SELECT count(*) FROM attachments WHERE id IN (%s) AND staging_key IS NOT NULL" % ids)[0][0] + limit = len(consumers) + by_name = {} + for s in samples: + by_name[s["name"]] = max(by_name.get(s["name"], 0), mib(s["used"])) + result = { + "count": args.count, "image_bytes": len(images[0]), "upload_seconds": round(t_up, 2), + "total_seconds_to_all_ready": round(total, 2), + "throughput_jobs_per_s": round(args.count / total, 2), + "consumers": limit, "prefetch_per_consumer": sorted({x["prefetch_count"] for x in consumers}), + "max_concurrent_processing": peak, + "queue_lag_s": {"p50": pct(lag, 50), "p95": pct(lag, 95), "max": round(max(lag), 3)}, + "enqueued_to_ready_s": {"p50": pct(to_ready, 50), "p95": pct(to_ready, 95), "max": round(max(to_ready), 3)}, + "processing_s": {"p50": pct(work, 50), "max": round(max(work), 3), "mean": round(statistics.mean(work), 3)}, + "worker_peak_mem_mib": {k: round(v, 1) for k, v in by_name.items()}, + "worker_mem_limit": samples[0]["limit"] if samples else None, + "staging_left": int(staging_left), + } + check(peak <= limit, "concurrency exceeded consumers x prefetch") + check(result["staging_left"] == 0, "staging not cleaned") + log.step("result", **result) + log.save(args.output, result=result) + print("PASS: %d jobs, max concurrent %d <= %d consumers" % (args.count, peak, limit)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/cursor_validation.py b/student-tests/cursor_validation.py new file mode 100644 index 0000000..25a9801 --- /dev/null +++ b/student-tests/cursor_validation.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Проверка диапазона дат cursor: некорректная дата должна давать 400, а не 500.""" +import base64 +import json +import sys +import uuid +from pathlib import Path +from urllib.parse import quote + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, ca_file=args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + checks = 0 + for kind, scope, path in ( + ("messages", chat["id"], "/api/v1/chats/" + chat["id"] + "/messages"), + ("chats", alice["id"], "/api/v1/chats"), + ): + for timestamp, expected in ((-10**17, 400), (-62135596800000001, 400), (0, 200)): + payload = {"k": kind, "s": scope, "t": timestamp, "i": str(uuid.uuid4())} + cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + client.call("GET", path + "?cursor=" + quote(cursor, safe=""), expected, actor=alice) + checks += 1 + print("PASS: %d cursor date boundary checks; invalid dates return 400." % checks) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/dashboard_check.py b/student-tests/dashboard_check.py new file mode 100644 index 0000000..4e0688f --- /dev/null +++ b/student-tests/dashboard_check.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Lab6: dashboard импортирован provisioning-ом и все его запросы работают на реальных данных. + + python3 student-tests/dashboard_check.py --output evidence/dashboard-check.json + +* Grafana отдаёт dashboard uid=messenger-overview (provisioned, папка Messenger) и 3 datasource; +* каждый PromQL панели выполняется через Grafana (ошибка синтаксиса/имени = FAIL); для + ключевых панелей результат обязан быть непустым (метрика реально существует); +* LogQL и TraceQL панелей выполняются в Loki/Tempo без ошибок. +Перед проверкой — прогрев 40 с (сообщения, чтение, upload до ready): панели с rate()/increase() +за 1 мин пусты, если за последнюю минуту не было трафика (например, сразу после пересоздания worker). +""" +import argparse +import json +import sys +import time +from pathlib import Path + +import obslib +from lab5lib import SAMPLE_PNG, Client, upload, wait_status +from traffic import Traffic + +ROOT = Path(__file__).resolve().parents[1] +# Панели, данные которых есть всегда после smoke-трафика (остальные — ошибки/алерты, могут быть пусты). +MUST_HAVE_DATA = { + "RPS бизнес-запросов", "p95 API (5m)", "RPS по маршруту", "Ответы по классу статуса", + "Задержка API p50/p95/p99 (из histogram)", "p95 по маршруту", "Запросы в обработке (saturation API)", + "HAProxy: UP-реплики и очередь LB", "Задания по исходу (в минуту)", "Глубина очереди: PostgreSQL и RabbitMQ", + "Возраст старейшего задания и lag", "Saturation worker: занятость потребителей", + "Пул соединений API: used / idle / max", "Операции БД p95 по операции (API)", "S3 p95 по операции", + "CPU, ядер (cgroup контейнера)", "Память контейнеров приложений", "Память: доля лимита", + "Память telemetry-стека (RSS)", "Collector: принято / отправлено / потеряно (spans)", "Collector: логи", + "Collector: очередь экспортёров", "SDK: заполнение очереди BatchSpanProcessor", "Time to ready p95", + "Ready lag очереди", "Relay: публикации outbox", "Ожидание в очереди p95 (до claim)", +} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--output") + p.add_argument("--warmup", type=int, default=40) + args = p.parse_args() + if args.warmup: + c = Client(timeout=30) + alice, _bob, _eve, chat = c.bootstrap() + tr = Traffic(c, alice, chat, threads=2, think=0.3, upload_every=3).start() + att = upload(c, alice, chat, SAMPLE_PNG, "warmup.png") + wait_status(c, alice, att["id"], ("ready",), timeout=60) + time.sleep(args.warmup) + tr.stop() + print("warmup:", tr.window(0, args.warmup + 60)) + dash = json.loads((ROOT / "infra/grafana/dashboards/messenger.json").read_text(encoding="utf-8")) + got = obslib.grafana("/api/dashboards/uid/" + dash["uid"]) + assert got and got["dashboard"]["uid"] == dash["uid"], "dashboard is not provisioned" + meta = got["meta"] + ds = sorted(d["uid"] for d in obslib.grafana("/api/datasources")) + assert {"prometheus", "loki", "tempo"} <= set(ds), ds + health = {uid: obslib.grafana("/api/datasources/uid/%s/health" % uid) for uid in ("prometheus", "loki", "tempo")} + report = {"dashboard": {"uid": dash["uid"], "title": got["dashboard"]["title"], "provisioned": meta.get("provisioned"), + "folder": meta.get("folderTitle"), "panels": len(dash["panels"])}, + "datasources": ds, "datasource_health": {k: (v or {}).get("status") for k, v in health.items()}, + "panels": []} + failed = [] + now = time.time() + for panel in dash["panels"]: + if panel["type"] == "row": + continue + for t in panel.get("targets", []): + uid = t["datasource"]["uid"] + item = {"panel": panel["title"], "ref": t["refId"], "datasource": uid} + try: + if uid == "prometheus": + expr = t["expr"].replace('instance=~"$instance"', 'instance=~".*"') + res = obslib.prom_query(expr) + item["series"] = len(res) + elif uid == "loki": + item["lines"] = len(obslib.loki_query(t["expr"], now - 3600, now, limit=20)) + else: + r = obslib.tempo_search(t["query"], now - 3600, now, limit=5) + item["traces"] = len((r or {}).get("traces") or []) + item["ok"] = True + except Exception as exc: + item["ok"], item["error"] = False, str(exc)[:200] + failed.append(item) + report["panels"].append(item) + empty_required = sorted({i["panel"] for i in report["panels"] if i["panel"] in MUST_HAVE_DATA + and i.get("series", 1) == 0 and i["ref"] == "A"}) + report["query_errors"] = failed + report["empty_required_panels"] = empty_required + report["pass"] = not failed and not empty_required and report["datasource_health"] == { + "prometheus": "OK", "loki": "OK", "tempo": "OK"} + for i in report["panels"]: + print("%-4s %-55s %-10s %s" % ("ok" if i["ok"] else "ERR", i["panel"][:55], i["datasource"], + i.get("series", i.get("lines", i.get("traces", i.get("error")))))) + print(json.dumps({k: report[k] for k in ("dashboard", "datasource_health", "query_errors", + "empty_required_panels", "pass")}, ensure_ascii=False)) + if args.output: + Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + return 0 if report["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/drain_bench.py b/student-tests/drain_bench.py new file mode 100644 index 0000000..de4a65b --- /dev/null +++ b/student-tests/drain_bench.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Lab6: скорость разбора backlog worker-ами (до/после исправления цикла AMQP в worker.run_job). + + python3 student-tests/drain_bench.py --count 240 --label after-fix --output evidence/drain-bench-after.json + +worker останавливаются, через API принимаются --count маленьких PNG (64x48, 202 queued), worker +запускаются; измеряется время, пока все вложения не станут ready, и по Prometheus — пропускная +способность (jobs/s), utilization потребителей и CPU worker в окне разбора. +""" +import argparse +import concurrent.futures +import sys +import time + +from lab5lib import Client, Failure, Log, check, compose, png, sql, upload +import obslib + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--count", type=int, default=240) + p.add_argument("--label", default="run") + p.add_argument("--output") + args = p.parse_args() + log = Log("worker drain benchmark (%s)" % args.label) + c = Client(timeout=30) + alice, _bob, _eve, chat = c.bootstrap() + compose("stop", "worker") + imgs = [png(64, 48, color=(i % 256, (i * 7) % 256, 90)) for i in range(args.count)] + with concurrent.futures.ThreadPoolExecutor(8) as ex: + ids = list(ex.map(lambda img: upload(c, alice, chat, img, "bench.png")["id"], imgs)) + log.step("backlog created with workers stopped", uploads=len(ids)) + t0 = time.time() + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", timeout=200) + t_up = time.time() + while True: + left = int(sql("SELECT count(*) FROM attachments WHERE chat_id = '%s' AND status <> 'ready'" % chat["id"])[0][0]) + if left == 0: + break + check(time.time() - t0 < 600, "drain timeout") + time.sleep(0.5) + t1 = time.time() + drain = t1 - t_up + time.sleep(25) # метрики экспортируются раз в 10 с + w = max(30, int(t1 - t_up) + 15) + res = { + "uploads": len(ids), "drain_seconds_after_workers_up": round(drain, 1), + "jobs_per_second_wallclock": round(len(ids) / drain, 2), + "workers_x_concurrency": obslib.prom_value("sum(last_over_time(messenger_worker_capacity[30s]))"), + "utilization_peak_1m": round(obslib.prom_value( + "max_over_time((sum(rate(messenger_worker_busy_time_seconds_total[30s])) / sum(last_over_time(messenger_worker_capacity[30s])))[%ds:5s])" % w), 3), + "job_duration_p50_s": round(obslib.prom_value( + 'histogram_quantile(0.5, sum by (le) (increase(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[%ds])))' % w), 3), + "worker_cpu_cores_peak": round(obslib.prom_value( + 'max_over_time(sum(rate(container_cpu_time_seconds_total{job="messenger-worker"}[30s]))[%ds:5s])' % w), 3), + } + log.step("drained", **res) + log.save(args.output, label=args.label, **res) + print("PASS: %s drained %d jobs in %.1fs (%.1f jobs/s)" % (args.label, len(ids), drain, len(ids) / drain)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + compose("up", "-d", "--no-deps", "--no-recreate", "worker", ok=False) + sys.exit(1) diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c21ab744523d9dc47b8339f29cd0a734004db6fe GIT binary patch literal 8265 zcmeHLZCF!Rw%&*o3mp*=P^)AF5doc4se>QrX;rX-(^ODUR5D7HR$3XPR0L#KXD9*+ z7X+hKi65nuDx?;Xs=(0#1`t6l8WlBs1W*u30)d=xvd^s8>IjeA=idA8K2O5~Kf+1& zUTeMYyWX|9yIdn0VlKGKH6gx%r!K zSz3%8YiVI=X=O9X)@s~DYfH=VQ^rr6JekMiS=ri8wYQr($&P1d_!1-dsOcM~qeqPz zZ8y$xoZbKW#a%|0qtJ1rG&ZtFBP@-KEseNx#Dja97(IWX-(N-}j7P#PN4;rg4i6NL zK_iTejYq)!Apm%K7yKWMv@{tzW!C#|SgqV>Y9Bss_U^P3qZ}5Wy<+WqQ#sXb)Al`Y zn%UTnpD@vJn$vV=mv`s<)qU=~`Ae31dM*24xwqe{)&2o%)&>TDym?E=C!cPO_#*Pl z9Z@@@|M{=@uM_qrCZ!+9$UG=IboiT-r@qbpF6aB)AATx0S9reY=L;8qDJ!qIDy^)l zzV&PU?K`r&_wF|}w zNA^DpY|sBGvX_DVC$9VGEn_1np0OpOkm~-8a~w*IWbv_(JMlSejHHF_lr?qtAI9Tm1C$YJer(RP>N>{Vrt*z%!%T+-clT@BpA|M_gv(%w8-s>GmD_=!bPt~S!$iW+H zy6_X<1<|dAa>ndBhm1IM!!vTuwNP4ZeS{cu=y($I=uaGq6Ki?xR*#Dpb13aI>MGrP zc;cfsIV7UBrsR5H&@>MH*JLSc%Av*9M;gUU42QnLPj6>8hcQME@MYXPWz80>_LZ_< zaA^9`djV{^nnNan+GSQjMM2AhiZ*2Z8P`zy5)PfuqDJ6r92(fcp=}=BS1I+Zhxjtp zYm)!ZrVzHA?h=u@^&HxV6~Pw`hdeglVERiPlZYGm`m9R@sf=TQ)|rHR)R?jvz8sn? z(Et2EC1;aFVx@(Cd|_*5N8AF4!MBG%sHbd-e-b5dNIg@mm6HQwqojHruSs|`_4sLN zsXd1(TQP?|<4|v)>_=H@tS5Z1nnUCshaMM7(v{uBE0Vel|NcHJ$C^VIq^yrR#80_| zY~oN~8Hct8_^2qd^z&up9)4V(AKI9}R&b~*fz;=VnlZcgeyaW_4&BUM(@v?^NJ*(L zdpx3%R;=Jq8C9<`(`A*ek>qyXPzlIJ+{GdNk3CU4NKh>8ce!*1hpOI#0z*2Iyq|I? zgkn;015Iw{yEWIb8GjNOj{G*it}lW^hx+xG;K@LZ8;2PCV1V0`xPQ$39#=xWlZ3gl zSa>Tv{0Bf9Pd_QMCwrHzDwQn1dKyB` zrF7@Ru-f3$iwimz7dFcoxNBHk!fvQ@1#S=z?EJ7@9}&l^mV6>sHxOg}*8O$5HkQf^ zx=FPbv8Fni|7JMEnv9PN_gH_5_)OrCteVcaHebv}RcZ08HJL@Z9Fp!6C-hh+`uqz& z$RM&zJn|-FyPfvxZn!i{VRuz+2&og zdxh$m30fIB;IQrrrS9jTr=WYMeahu(U;uwbbl#M>Wu~6$nA_G$Hb&6G~?7kgBIK?RRxOOdvX7L9g2?eq*!y%tgLKCXi=`dU< zJxSOC!56)qH5lzB0x4Fz1Q{5$Np{dz3262#;1Ja1)+WJt4$(_k*PgUq>!|*6A!rm_ zMof|e`zETo_tNTlLcj>6w<%NyO$Kt*=4g2&)2hCi?w`w{#JCk5aiegJkle*x-cRD3 z;EDa;!w2~KiJ8}9$3PIY{zneo&%DJ708ekP1hNZtb6h926CXHPRxQjJ9c#-binSrS ztSLf2*)?m;!e~2Z4%P7WG(2natGT+@OT+vy668|{^}Vy{RalgHFVe&WCD6aV6M z9Qq6H>b{}$FcexcU|@kW=ze}!8MC^}imU@X=B>QIq4n|v_38Ug?D^-A3+$+&6^rRI zi(04T3|tjV>6ifZ;SfJ)M-wQEHrKJ|HI!C$KuI>8b^b@{Kn7h7(tDtpV)sKvWrZBd z{-CS#pbRTq-3_|^zIDB9WZwF7w(~t5G+k;!X4T@UeFG{2prhrC6CyC#97;@>uQ~k=GjMywx`&?tK zsdqjq!X9gxa?azBcF6X*kQlk0!(p#ct+(#YpDyJ2(DB3|4x)&Q1) zg%O70q@{EkjygW`3m)`HN02GHFZx+6rN|UE<}l?w@9@+seb`8bu4vuk7HXke*MjDkvr% z&}--?*PGqd3WbI~x(#==E)Z+3Yq3ko zNqi0PsQJNthy~iPR>bZN&~9YPRohdU8BiLb3-O!LfmJO|O>D&`|3^6-vYQ>ptMPaI zizSD)i?rg_v9Ze_$JvywkM7N#T(bRsHygj@H~qH1_{|^-6x;m#>iQLh4BLm9+6cjt3D-AoQz z>r)2sw5emmz?3F>1QWf5Qc}C@#_ATir9q?sS6DYQ^ls=-3se;a3hOXm+=z9n1b#^z zdeA}~8Mj>R6p&v$un|D!nynQ>PyXvE11WpA2Fw}G44sJ(Gc$srYR+*V%~J~1({Pix zBPxEEKsY0yUh_B#s`n1v2i@uJ&SL0b{6)gtMxhN>9N&r!`S5i75*LVBDcvOpnO`GTH1X%(e_f>-eCJ4@lMS5&?k2CqvOt0=a~z%{X!{oXV(h`l%zm&*(uTr+fwhd zYXDsf`8qAQ<8*freeZ|4wt2z(;oWKujiZ)`!BVE%3pLM*+GpK9u~u^;P!+~rfU46= zE*nZsch%{Ob0WwE+M+e!oPhZpRFo8Y4R%4Hr)R?#nt@PIbF==k5~N@shdNt@)tisK zH2DpnPrqCL>|dy7z8tb*PYU*e^t=m63X+jbhoby-8^E-mk%NAZ4Dfr^2T6`K%KM*i zXg3L8lw8hdY|7YBfJ$24YmNQdQc&*K%>iGBz};qGsX(20Jrh^)`N~zSB79C~q$0C3 z@?Ntk3ovYu4}999bYcN?MYyk}mIX*SOqLl4_FmxQMK-v`2aN1fP2F&Y=n^=O^woHe zxZSGeGOqIEt044-{dGeiAC%KVGuC_xXf-SJnsXEekly1El;+7Sr+rv{f><+^I(gKZGQ2LqxA@nbDGCTcfMN?`ZNwb{C3I|fU9z)cJdS8;SJ-3J_4<{10L z=NCCMZWVAQ73%&~fuuzuxP5r_iBCaNJGC5|U-yCtva@CKZos2CdvidT%u_pM1Ep=r zf!SUl`Jb*UwAY*oH~7ZpgvSvIp?awgDY?V8%|GY5=$vcNOu!A**Z(-m4(c4BHxMD? z*^tTh`=LfCa*I$sMc9Ye`Riv|NDEZw z^>%1S2v6-TQTnmXI}cef)3eEZ@j&>6SE24&&w^;%XSW+DD zsRO;#FZ4X#`vF6T+l@nwkkGM7&hG2LyMZz*7k=owiNKu1k3SqCYg20B23 z)JvtL-a4^_Q3uCau-H9h2)rX)H+C7l31gHXy37kWFw#G z3S7?3q*@+gUONc)OD6QqjnTIWoA5VOs4AE&?WH@L0c2Y~Fetw(?0X7R+`&AuAL&QO zCX){mvFxJ!sMJ3mmPMfzw7Q)<(pwnA`U0A15!I(YHh93j{q|rNlz-+LS?bEZ!!}1< zSjHZ71A=B^=Yx?ie755YZmOk*F+)#V$nw?142BaoBhb-tv=jp0=I${V z>e1p9!1}7<=WC)0$&BnKumZ^;#2t)jc^35lxCCuLyTfCb!Ncv-IfIRm!eBtsjtP{M z-o+`m0zV7q&{+0xJv(ODk@lP6Es_7cI`$x(Eytl zq3gc70OFft#Y#U?O6jMKsHIa%DF4m=FP58&Ua%lYYJrQdvHYH8U2!vTWtK=E3l6#J zZP5H|!@}q6Re)V9PU4-YOT$%`d|m&#oHRz*4edSX?}P5pTG3~zD>M-XPih@I+bH)A|zD; z7REcRVbF}5JIDjoD#HeP!NY@>7JD|^vdJ6C96(Cto!5JN`7onEnHyZ~Ak~t?EN!c% z6(W(=C2srNZ!{Ox-kOU~RmKE-a#y$hoJwm&gpdV#AiO;e0P)(XwtYB5^zbOIa-pvX zdJ^0IPAN``v>mwXYWrZkOZ;A4d}b7b6-fJxLr|CALkF%>-8VR7#wvE144H6Cz8+$O zVnqp9?UqYSVP~*ZGgqRorJ98Y=8K0NYa@ii86xRCL)|EiTGVPRHUW(dArnZ>Ua Q!9Tnk`v14!r?`9n1|_}m^8f$< literal 0 HcmV?d00001 diff --git a/student-tests/incident_db_slow.py b/student-tests/incident_db_slow.py new file mode 100644 index 0000000..eea8ed3 --- /dev/null +++ b/student-tests/incident_db_slow.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Lab6, инцидент 2: медленная БД (долгая блокировка горячей таблицы) против медленного обработчика. + + python3 student-tests/incident_db_slow.py --output evidence/incident-db-slow.json + +Методика: + 0. API пересоздаётся с FAULT_INJECTION=1 (точка slow-handler доступна только по заголовку + X-Fault-Inject; обычные запросы не меняются). Эталон «медленный КОД»: 3 запроса истории с + X-Fault-Inject: slow-handler (sleep 1.5 с в span handler.compute, БД не трогается). + 1. Нагрузка traffic.py (4 потока, история + сообщения, без upload), 40 с базовой линии. + 2. ОТКАЗ: отдельная сессия psql (как «забытая» миграция) берёт LOCK TABLE sessions IN ACCESS + EXCLUSIVE MODE и держит его. Каждый аутентифицированный запрос ждёт блокировку до + statement_timeout (5 с) или PoolTimeout (3 с) -> 503. Readiness (SELECT 1 отдельным + соединением) остаётся 200: реплики в LB, отказ виден в метриках API, а не LB. + 3. Ждём firing MessengerHighErrorRate{source=api} и MessengerDbSlow; через 20 с — снятие + блокировки (pg_terminate_backend); ждём resolved обоих. +Диагностика по сохранённым наблюдениям: trace ошибки (длинный db span, короткий остальной код) +vs trace slow-handler (длинный handler.compute, короткие db spans); гистограммы +db.client.operation.duration и http.server.request.duration, пул (used/pending), логи +event=db.unavailable с trace_id, pg_stat_activity (wait_event_type=Lock). +""" +import argparse +import subprocess +import sys +import time + +from lab5lib import ROOT, Client, Failure, Log, check, recreate, sql +from traffic import Traffic +import obslib + +ALERTS = ("MessengerHighErrorRate", "MessengerDbSlow") + + +def alert_states(): + out = {} + for a in obslib.prom_alerts(): + name = a["labels"].get("alertname") + key = name + ("{source=%s}" % a["labels"]["source"] if "source" in a["labels"] else "") + if out.get(key) != "firing": + out[key] = a["state"] + return out + + +def series(expr, start, end, step=15): + out = {} + for r in obslib.prom_range(expr, start, end, step): + key = ",".join("%s=%s" % kv for kv in sorted(r["metric"].items()) if kv[0] != "__name__") or "value" + out[key] = [[round(float(t) - start), round(float(v), 4)] for t, v in r["values"] if v not in ("NaN",)] + return out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + p.add_argument("--baseline", type=int, default=40) + args = p.parse_args() + log = Log("incident 2: slow database (table lock) vs slow handler code") + recreate("api", {"FAULT_INJECTION": "1"}) + time.sleep(6) # HAProxy видит новые реплики (DNS hold 2 с, rise 2) + log.step("api recreated with FAULT_INJECTION=1 (slow-handler only by header)") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + for i in range(3): + c.message(alice, chat, "warmup %d" % i) + + # ---- эталон «медленный обработчик» + slow = [] + for _ in range(3): + r = c.call("GET", "/api/v1/chats/%s/messages?limit=20" % chat["id"], actor=alice, + headers={"X-Fault-Inject": "slow-handler"}) + slow.append(r["headers"]["X-Trace-Id"]) + normal = c.call("GET", "/api/v1/chats/%s/messages?limit=20" % chat["id"], actor=alice)["headers"]["X-Trace-Id"] + slow_spans = obslib.wait_trace(slow[0], lambda s: any(x["name"] == "handler.compute" for x in s)) + normal_spans = obslib.wait_trace(normal, lambda s: any(x["name"].startswith("db ") for x in s)) + hc = [s for s in slow_spans if s["name"] == "handler.compute"][0] + db_ms = sum(s["duration_ms"] for s in slow_spans if s["name"].startswith("db ")) + log.step("reference: slow handler trace", trace_id=slow[0], server_ms=slow_spans[0]["duration_ms"], + handler_compute_ms=hc["duration_ms"], db_spans_total_ms=round(db_ms, 1)) + log.step("alerts before experiment", states=alert_states()) + + t_wall0 = time.time() + tr = Traffic(c, alice, chat, threads=4, think=0.2, upload_every=0).start() + time.sleep(args.baseline) + base = tr.window(5, args.baseline) + base_db_p95 = obslib.prom_value("messenger:db_operation_seconds:p95_2m") + log.step("baseline", db_p95_s=base_db_p95, **base) + + # ---- отказ: блокировка + t_lock = time.time() + lock = subprocess.Popen( + ["docker", "compose", "exec", "-T", "db", "sh", "-c", + 'su-exec postgres psql -U postgres -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -q'], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True) + lock.stdin.write("BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;\n") + lock.stdin.close() + log.step("FAULT: LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE held by a separate session") + waits, fired, seen = None, {}, {} + deadline = time.monotonic() + 300 + while time.monotonic() < deadline and not all(k in fired for k in ("MessengerHighErrorRate{source=api}", "MessengerDbSlow")): + st = alert_states() + for k, v in st.items(): + if seen.get(k) != v: + log.step("alert %s -> %s" % (k, v)) + seen[k] = v + if v == "firing" and k not in fired: + fired[k] = round(time.time() - t_lock, 1) + if waits is None and time.time() - t_lock > 20: + waits = sql("SELECT coalesce(wait_event_type,'-'), coalesce(wait_event,'-'), count(*) FROM pg_stat_activity " + "WHERE datname = current_database() AND state = 'active' AND pid <> pg_backend_pid() GROUP BY 1, 2 ORDER BY 3 DESC") + log.step("pg_stat_activity during incident", active_by_wait=[[w[0], w[1], int(w[2])] for w in waits]) + time.sleep(5) + check("MessengerHighErrorRate{source=api}" in fired and "MessengerDbSlow" in fired, "alerts fired: %s" % fired) + snap = { + "api_5xx_ratio": obslib.prom_value("messenger:http_error_ratio:rate2m"), + "db_p95_s": obslib.prom_value("messenger:db_operation_seconds:p95_2m"), + "db_p95_by_op_s": {r["metric"].get("db_operation_name"): round(float(r["value"][1]), 3) for r in obslib.prom_query( + 'histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job="messenger-api"}[1m])))') + if r["value"][1] != "NaN"}, + "pool_used": obslib.prom_value('sum(db_client_connection_count{job="messenger-api", db_client_connection_state="used"})'), + "pool_pending": obslib.prom_value('sum(db_client_connection_pending_requests{job="messenger-api"})'), + "error_types_rps": {r["metric"].get("error_type"): round(float(r["value"][1]), 2) for r in obslib.prom_query( + 'sum by (error_type) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))')}, + "lb_up_servers": obslib.prom_value('haproxy_backend_active_servers{proxy="be_api"}'), + } + log.step("alerts firing", detect_s=fired, **snap) + time.sleep(20) + during = tr.window(t_lock - t_wall0, time.time() - t_wall0) + log.step("client view during lock", **during) + + # ---- восстановление + t_release = time.time() + killed = sql("SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE query ILIKE '%pg_sleep(600)%' " + "AND pid <> pg_backend_pid()") + lock.wait(timeout=30) + log.step("RECOVERY: lock holder terminated", terminated=int(killed[0][0]), lock_s=round(t_release - t_lock, 1)) + resolved = {} + deadline = time.monotonic() + 400 + while time.monotonic() < deadline and len(resolved) < len(fired): + st = alert_states() + for k in fired: + if k not in resolved and st.get(k, "inactive") == "inactive": + resolved[k] = round(time.time() - t_release, 1) + log.step("alert %s resolved" % k, after_release_s=resolved[k]) + time.sleep(5) + check(len(resolved) == len(fired), "alerts resolved: %s" % resolved) + time.sleep(10) + tr.stop() + after = tr.window(time.time() - t_wall0 - 10, time.time() - t_wall0) + log.step("after recovery", **after) + + # ---- трасса и логи ошибки + found = obslib.tempo_search('{ resource.service.name = "messenger-api" && kind = server && status = error }', + t_lock, t_release + 5, limit=10) + tids = [t["traceID"] for t in (found or {}).get("traces", [])] + check(tids, "error traces in Tempo for the lock window") + err_spans = obslib.wait_trace(tids[0]) + db_err = [s for s in err_spans if s["name"].startswith("db ")] + worst = max(db_err, key=lambda s: s["duration_ms"]) if db_err else None + server_ms = max(s["duration_ms"] for s in err_spans) + log.step("error trace during lock", trace_id=tids[0], server_ms=server_ms, + slowest_db_span=worst and worst["name"], slowest_db_ms=worst and worst["duration_ms"], + db_share=round(sum(s["duration_ms"] for s in db_err) / server_ms, 3) if db_err else None, + error_type=[s["attrs"].get("error.type") for s in err_spans if s["kind"] == "SPAN_KIND_SERVER"]) + dlogs = obslib.loki_query('{service_name="messenger-api"} | event="db.unavailable"', t_lock, t_release + 10, limit=500) + sample = dlogs[0] if dlogs else {} + log.step("API logs event=db.unavailable (Loki)", count=len(dlogs), + sample_trace_id=sample.get("labels", {}).get("trace_id"), sample_error_type=sample.get("labels", {}).get("error_type"), + sample=sample.get("line", "")[:120]) + recreate("api") + log.step("api restored with FAULT_INJECTION=0") + end = time.time() + evidence = { + "timeline_s": {"lock_taken": 0, "alerts_firing": fired, "lock_released": round(t_release - t_lock, 1), + "alerts_resolved_after_release": resolved}, + "snapshot_firing": snap, + "pg_stat_activity": waits, + "traffic_by_10s": tr.summary_by(10), + "trace_slow_handler": obslib.tree(slow_spans), + "trace_normal": obslib.tree(normal_spans), + "trace_db_lock_error": obslib.tree(err_spans), + "prometheus": { + "api_5xx_ratio": series("messenger:http_error_ratio:rate2m", t_wall0, end), + "lb_5xx_ratio": series("messenger:lb_error_ratio:rate2m", t_wall0, end), + "db_p95_s": series("messenger:db_operation_seconds:p95_2m", t_wall0, end), + "http_p95_s": series("messenger:http_latency_seconds:p95_5m", t_wall0, end), + "pool_used": series('sum(db_client_connection_count{job="messenger-api", db_client_connection_state="used"})', t_wall0, end), + "pool_pending": series('sum(db_client_connection_pending_requests{job="messenger-api"})', t_wall0, end), + "pool_wait_p95_s": series('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{job="messenger-api"}[1m])))', t_wall0, end), + "alerts": series('max by (alertname, source, alertstate) (ALERTS{alertname=~"MessengerHighErrorRate|MessengerDbSlow|MessengerDbPoolSaturated|MessengerApiLatencyHigh"})', t_wall0, end), + }, + } + log.save(args.output, **evidence) + print("PASS: DB lock detected %s; resolved after release %s" % (fired, resolved)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + try: + sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query ILIKE '%pg_sleep(600)%' AND pid <> pg_backend_pid()") + recreate("api") + except Exception: + pass + sys.exit(1) diff --git a/student-tests/incident_worker_stop.py b/student-tests/incident_worker_stop.py new file mode 100644 index 0000000..5a6f73c --- /dev/null +++ b/student-tests/incident_worker_stop.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Lab6, инцидент 1: остановка всех worker под нагрузкой -> рост возраста заданий -> alert -> восстановление. + + python3 student-tests/incident_worker_stop.py --output evidence/incident-worker-stop.json + +Методика (повторяемая): смешанная нагрузка traffic.py (4 потока: история + сообщение, каждый +4-й цикл — upload 64x48 PNG); 45 с базовой линии; `docker compose stop worker`; ждём firing +MessengerQueueLagGrowing (ready lag > 30 с, не убывает, for 1m); ещё 20 с; `docker compose up worker`; +ждём, пока backlog = 0 и alert снят (resolved). Диагностика — ТОЛЬКО по сохранённым наблюдениям: +ряды Prometheus (lag, backlog, RabbitMQ ready, processed по исходам, time to ready), события +alerts (ряд ALERTS), трасса upload из окна отказа (разрыв send -> process), логи worker в Loki. +""" +import argparse +import sys +import time + +from lab5lib import Client, Failure, Log, check, compose, sql +from traffic import Traffic +import obslib + +ALERT = "MessengerQueueLagGrowing" + + +def alert_state(name): + states = [a["state"] for a in obslib.prom_alerts() if a["labels"].get("alertname") == name] + return "firing" if "firing" in states else ("pending" if "pending" in states else "inactive") + + +def wait_alert(name, wanted, timeout, log, every=5): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + st = alert_state(name) + if st != last: + log.step("alert %s -> %s" % (name, st)) + last = st + if st in wanted: + return st + time.sleep(every) + raise Failure("alert %s did not reach %s in %ss (last %s)" % (name, wanted, timeout, last)) + + +def series(expr, start, end, step=15): + out = {} + for r in obslib.prom_range(expr, start, end, step): + key = ",".join("%s=%s" % kv for kv in sorted(r["metric"].items()) if kv[0] != "__name__") or "value" + out[key] = [[round(float(t) - start), round(float(v), 3)] for t, v in r["values"]] + return out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + p.add_argument("--baseline", type=int, default=45) + args = p.parse_args() + log = Log("incident 1: all workers stopped under load") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + check(alert_state(ALERT) == "inactive", "alert must be inactive before the experiment") + t_wall0 = time.time() + tr = Traffic(c, alice, chat, threads=4, think=0.2, upload_every=4).start() + log.step("traffic started", threads=4, think_s=0.2, upload_every=4) + time.sleep(args.baseline) + base = tr.window(5, args.baseline) + log.step("baseline", **base) + + t_stop = time.time() + compose("stop", "worker") + log.step("FAULT: docker compose stop worker", + running=compose("ps", "-q", "--status", "running", "worker").stdout.split()) + wait_alert(ALERT, ("firing",), 300, log) + t_fire = time.time() + lag_at_fire = obslib.prom_value("max(messenger_jobs_ready_lag_seconds)") + backlog_at_fire = obslib.prom_value('max(messenger_jobs_count{messenger_job_status="queued"})') + rmq_ready = obslib.prom_value('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})') + log.step("alert firing", detect_s=round(t_fire - t_stop, 1), ready_lag_s=lag_at_fire, + queued_jobs_db=backlog_at_fire, rabbitmq_ready=rmq_ready) + time.sleep(20) + outage_api = tr.window(t_stop - t_wall0, time.time() - t_wall0) + log.step("API during worker outage", **outage_api) + + t_start = time.time() + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", timeout=200) + log.step("RECOVERY: docker compose up worker", outage_s=round(t_start - t_stop, 1)) + + # Снятие alert и опустошение backlog фиксируются независимо: alert снимается, как только + # lag перестаёт расти (система восстанавливается), backlog может дренироваться дольше. + t_drain = t_resolved = None + deadline = time.monotonic() + 420 + while (t_drain is None or t_resolved is None) and time.monotonic() < deadline: + if t_resolved is None and alert_state(ALERT) == "inactive": + t_resolved = time.time() + log.step("alert resolved", after_worker_start_s=round(t_resolved - t_start, 1), + ready_lag_s=obslib.prom_value("max(messenger_jobs_ready_lag_seconds)")) + if t_drain is None and (obslib.prom_value("max(messenger_jobs_ready_lag_seconds)", default=99) or 0) < 5: + t_drain = time.time() + log.step("backlog drained: ready lag < 5 s", after_worker_start_s=round(t_drain - t_start, 1), + queued_jobs_db=obslib.prom_value('max(messenger_jobs_count{messenger_job_status="queued"})')) + time.sleep(3) + check(t_drain and t_resolved, "recovery not observed") + time.sleep(15) + tr.stop() + after = tr.window(max(t_resolved, t_drain) - t_wall0, time.time() - t_wall0) + log.step("after recovery", **after) + + # ---- доказательства из хранилищ (только сохранённые наблюдения) + rows = sql("SELECT a.status, count(*) FROM attachments a WHERE a.chat_id = '%s' GROUP BY 1 ORDER BY 1" % chat["id"]) + statuses = {r[0]: int(r[1]) for r in rows} + check(set(statuses) == {"ready"}, "all uploads of the experiment are ready: %s" % statuses) + # трасса upload, принятого посередине отказа + mid = [aid for (t, aid) in tr.uploads if t_stop - t_wall0 + 10 <= t <= t_start - t_wall0 - 5] + check(mid, "uploads during outage") + tp = sql("SELECT o.payload->>'traceparent' FROM outbox o JOIN jobs j ON j.id = o.job_id " + "WHERE j.attachment_id = '%s' ORDER BY o.id LIMIT 1" % mid[len(mid) // 2])[0][0] + trace_id = tp.split("-")[1] + spans = obslib.wait_trace(trace_id, lambda s: any(x["name"] == "attachments.process.v1 process" for x in s)) + send = [s for s in spans if s["name"] == "attachments send"][0] + proc = [s for s in spans if s["name"] == "attachments.process.v1 process"][0] + queue_gap = round(proc["start"] - send["end"], 1) + log.step("trace of an upload accepted during outage", trace_id=trace_id, publish_to_consume_gap_s=queue_gap, + process_ms=proc["duration_ms"], queue_wait_attr_s=proc["attrs"].get("messenger.job.queue_wait_s")) + wlogs = obslib.loki_query('{service_name="messenger-worker"} | event=~"worker.(draining|stopped|started)"', + t_wall0, time.time(), limit=50) + log.step("worker lifecycle logs (Loki)", events=[(round(w["ts"] - t_stop, 1), w["labels"].get("event"), + w["labels"].get("service_instance_id")) for w in wlogs]) + end = time.time() + evidence = { + "timeline_s": {"worker_stopped": 0, "alert_firing": round(t_fire - t_stop, 1), + "worker_started": round(t_start - t_stop, 1), "backlog_drained": round(t_drain - t_stop, 1), + "alert_resolved": round(t_resolved - t_stop, 1)}, + "traffic_by_10s": tr.summary_by(10), + "attachments_final": statuses, + "trace_tree": obslib.tree(spans), + "prometheus": { + "ready_lag_s": series("max(messenger_jobs_ready_lag_seconds)", t_wall0, end), + "oldest_queued_age_s": series('max(messenger_jobs_oldest_age_seconds{messenger_job_status="queued"})', t_wall0, end), + "queued_jobs_db": series('max(messenger_jobs_count{messenger_job_status="queued"})', t_wall0, end), + "rabbitmq_ready": series('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', t_wall0, end), + "jobs_per_min_by_outcome": series("sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", t_wall0, end), + "time_to_ready_p95_s": series('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[1m])))', t_wall0, end), + "worker_utilization": series("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", t_wall0, end), + "worker_cpu_cores": series('sum by (instance) (rate(container_cpu_time_seconds_total{job="messenger-worker"}[1m]))', t_wall0, end), + "api_5xx_ratio": series("messenger:http_error_ratio:rate2m", t_wall0, end), + "alerts": series('max by (alertname, alertstate) (ALERTS{alertname="%s"})' % ALERT, t_wall0, end), + }, + "queries": { + "ready_lag": "max(messenger_jobs_ready_lag_seconds) [s]", + "backlog": 'max(messenger_jobs_count{messenger_job_status="queued"}) [jobs]', + "alert": "max(messenger_jobs_ready_lag_seconds) > 30 and deriv(max(messenger_jobs_ready_lag_seconds)[2m:15s]) >= 0, for 1m", + }, + } + log.save(args.output, **evidence) + print("PASS: worker outage detected in %.0fs, alert resolved %.0fs after restart, drained %.0fs after restart; API 5xx during outage %.2f%%" % ( + t_fire - t_stop, t_resolved - t_start, t_drain - t_start, 100 * outage_api.get("error_ratio_5xx", 0))) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + sys.exit(1) diff --git a/student-tests/janitor_replay_race.py b/student-tests/janitor_replay_race.py new file mode 100644 index 0000000..ea579d5 --- /dev/null +++ b/student-tests/janitor_replay_race.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Аудит F01: очистка staging janitor и операторский replay не теряют исходник принятой работы. + + python3 student-tests/janitor_replay_race.py --output evidence/janitor-replay-race.json + +Готовим три failed-вложения (fault-transient-always -> retry_exhausted; worker на время теста +с WORKER_FAULT_INJECTION=1). Перед каждым сценарием «старим» ровно одно вложение (updated_at +на 2 суток назад > STAGING_RETENTION_SECONDS), чтобы janitor считал его staging просроченным. + + C (опасное чередование из аудита): janitor уже выбрал кандидата (учебная пауза + JANITOR_PAUSE_AFTER_SELECT_SECONDS), в это время replay коммитится. Ожидаем: janitor под + блокировкой перепроверяет, видит queued и ПРОПУСКАЕТ (staging_skipped_changed); staging + не освобождён; задание доходит до ready. + A (replay -> janitor): replay принят, janitor вложение не трогает, ready. + B (janitor -> replay): janitor снимает staging_key и удаляет объект; replay получает + refused_staging_expired — принятой работы без исходника не возникает. +Инвариант во всех случаях: не бывает «replay принят, а staging удалён». +""" +import argparse +import json +import subprocess +import time + +from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, ops, recreate, sql, upload, wait_status, + ROOT) + +FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "4"} + + +def run_janitor(): + t = time.monotonic() + out = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once").stdout + return json.loads(out.strip().splitlines()[-1]), round(time.monotonic() - t, 2) + + +def replay(att_id): + proc = compose("exec", "-T", "janitor", "python", "/app/src/ops.py", "replay", att_id, "--reason", "race test", + ok=False) + return json.loads(proc.stdout)["replay"][0]["result"] + + +def age(att_id): + sql("UPDATE attachments SET updated_at = now() - interval '2 days' WHERE id = '%s'" % att_id) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + args = p.parse_args() + log = Log("janitor staging cleanup vs operator replay") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + recreate("worker", FAULT_ENV) + res = {} + try: + ids = [upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png")["id"] for _ in range(3)] + for i in ids: + st = wait_status(c, bob, i, {"failed", "ready"}, 120) + check(st["status"] == "failed" and st["error_code"] == "retry_exhausted", "expected retry_exhausted") + z, x, y = ids + log.step("three failed attachments prepared", ids=ids) + + # --- C: janitor выбрал кандидата -> replay закоммитился -> janitor удаляет? ----------- + # Это чередование, которое аудит воспроизвёл на старом коде (исходник терялся). + age(z) + jn = subprocess.Popen(["docker", "compose", "exec", "-T", "-e", "JANITOR_PAUSE_AFTER_SELECT_SECONDS=4", + "janitor", "python", "/app/src/janitor.py", "--once"], + cwd=ROOT, stdout=subprocess.PIPE, text=True) + time.sleep(2.0) # janitor уже выбрал z и стоит в паузе + replay_out = replay(z) # replay коммитится между выборкой и захватом + jrep = json.loads(jn.communicate(timeout=60)[0].strip().splitlines()[-1]) + ready = wait_status(c, bob, z, {"ready", "failed"}, 60) + ev = events(ops("inspect", z)) + res["C_janitor_selected_then_replay"] = { + "attachment_id": z, "janitor_candidates_included_z": z in jrep["staging_candidates"], + "replay": replay_out, "janitor_staging_skipped_changed": jrep["staging_skipped_changed"], + "janitor_staging_failed_expired": jrep["staging_failed_expired"], + "final": ready["status"], "janitor_released_staging": "staging_released" in ev, "events": ev} + check(z in jrep["staging_candidates"] and replay_out == "replayed" + and jrep["staging_skipped_changed"] >= 1 and "staging_released" not in ev + and ready["status"] == "ready", "C: %s" % res["C_janitor_selected_then_replay"]) + log.step("C: janitor selected z, replay committed, janitor rechecked under lock and skipped -> ready", + **res["C_janitor_selected_then_replay"]) + + # --- A: replay, потом janitor -------------------------------------------------- + age(x) + r = replay(x) + jrep, _ = run_janitor() + info = ops("inspect", x) + res["A_replay_then_janitor"] = {"attachment_id": x, "replay": r, + "staging_object_present": bool(info["s3_staging_objects"]) or + info["attachment"]["status"] in ("processing", "ready")} + ready = wait_status(c, bob, x, {"ready", "failed"}, 60) + res["A_replay_then_janitor"]["final"] = ready["status"] + check(r == "replayed" and ready["status"] == "ready", "A: %s" % res["A_replay_then_janitor"]) + log.step("A: replay first -> janitor leaves it -> ready", **res["A_replay_then_janitor"]) + + # --- B: janitor, потом replay -------------------------------------------------- + age(y) + jrep, _ = run_janitor() + r = replay(y) + info = ops("inspect", y) + res["B_janitor_then_replay"] = {"attachment_id": y, "janitor_staging_failed_expired": jrep["staging_failed_expired"], + "replay": r, "status": info["attachment"]["status"], + "staging_key": info["attachment"]["staging_key"], + "staging_objects": info["s3_staging_objects"], + "events": events(info)[-3:]} + check(jrep["staging_failed_expired"] >= 1 and r == "refused_staging_expired" + and info["attachment"]["status"] == "failed", "B: %s" % res["B_janitor_then_replay"]) + log.step("B: janitor first -> replay refused, no job without source", **res["B_janitor_then_replay"]) + finally: + recreate("worker") + log.step("workers restored to .env configuration") + log.save(args.output, results=res) + print("PASS: janitor cleanup and replay never lose the source of accepted work") + + +if __name__ == "__main__": + main() diff --git a/student-tests/lab5lib.py b/student-tests/lab5lib.py new file mode 100644 index 0000000..9577a6d --- /dev/null +++ b/student-tests/lab5lib.py @@ -0,0 +1,175 @@ +"""Общие помощники проверок lab5 (запускаются на хосте, управляют стендом через docker compose). + +Токены сессий живут только в памяти процесса и не печатаются. В evidence пишутся ID, +статусы, времена и события заданий — без секретов и содержимого файлов. +""" +import datetime as dt +import json +import os +import random +import struct +import subprocess +import sys +import time +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402,F401 + +SAMPLE_PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def now(): + return dt.datetime.now(dt.timezone.utc).strftime("%H:%M:%S.%f")[:-3] + + +class Log: + def __init__(self, name): + self.name = name + self.steps = [] + self.started = time.monotonic() + + def step(self, text, **data): + item = {"t": round(time.monotonic() - self.started, 2), "utc": now(), "step": text} + item.update(data) + self.steps.append(item) + print("[%7.2fs] %s %s" % (item["t"], text, json.dumps(data, ensure_ascii=False) if data else ""), flush=True) + + def save(self, path, **extra): + if not path: + return + out = {"scenario": self.name, "date_utc": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), + "steps": self.steps} + out.update(extra) + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(json.dumps(out, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + + +def compose(*args, env=None, timeout=300, ok=True): + full_env = dict(os.environ) + full_env.update(env or {}) + proc = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, + env=full_env, timeout=timeout) + if ok and proc.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args[:3]), proc.stderr.strip()[-400:])) + return proc + + +def recreate(service, env=None): + """Пересоздать сервис с переопределёнными переменными (shell env важнее .env).""" + compose("up", "-d", "--no-deps", "--force-recreate", "--wait", service, env=env, timeout=400) + + +def sql(query): + proc = subprocess.run( + ["docker", "compose", "exec", "-T", "db", "sh", "-c", + 'su-exec postgres psql -U postgres -d "$POSTGRES_DB" -tA -F "|" -v ON_ERROR_STOP=1'], + cwd=ROOT, input=query, capture_output=True, text=True, timeout=60) + if proc.returncode != 0: + raise Failure("psql failed: " + proc.stderr.strip()[-300:]) + return [line.split("|") for line in proc.stdout.strip().splitlines() if line] + + +def ops(*args, ok=True): + proc = compose("--profile", "tools", "run", "--rm", "-T", "ops", *args, ok=False, timeout=120) + if ok and proc.returncode not in (0, 3): + raise Failure("ops %s failed: %s" % (args[0], proc.stderr.strip()[-400:])) + try: + return json.loads(proc.stdout) + except ValueError: + raise Failure("ops output is not JSON: " + proc.stdout[-300:] + proc.stderr[-300:]) + + +def queues(): + proc = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "-p", "messenger", "list_queues", "name", + "messages", "messages_ready", "messages_unacknowledged", "consumers", "--formatter", "json") + return {q["name"]: {k: q[k] for k in ("messages", "messages_ready", "messages_unacknowledged", "consumers")} + for q in json.loads(proc.stdout)} + + +def container_ids(service, running_only=True): + args = ["ps", "-q"] + (["--status", "running"] if running_only else []) + [service] + return [x for x in compose(*args).stdout.split() if x] + + +def upload(c, actor, chat, data, filename="photo.png", ctype="image/png", expected=202): + return c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], filename), expected, + actor=actor, raw=data, headers={"Content-Type": ctype}, + schema="Attachment" if expected == 202 else None)["json"] + + +def get_att(c, actor, att_id): + return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"] + + +def wait_status(c, actor, att_id, statuses, timeout=90, interval=0.3): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + att = get_att(c, actor, att_id) + if att["status"] in statuses: + return att + time.sleep(interval) + raise Failure("attachment %s did not reach %s in %ss (last %s)" % (att_id, statuses, timeout, att["status"])) + + +def wait_until(fn, timeout, interval=0.5, what="condition"): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = fn() + if last: + return last + time.sleep(interval) + raise Failure("timeout waiting for " + what) + + +def events(info): + return [e["event"] for e in info["events"]] + + +def invariants(info): + """Ключ эффекта: ровно 3 варианта и 4 конечных объекта на (attachment, v1), staging убран.""" + names = sorted(v["name"] for v in info["variants"]) + finals = [o[0] for o in info["s3_final_objects"]] + return { + "variants": names, + "variants_ok": names == ["crop", "optimized", "thumbnail"], + "final_objects": len(finals), + "final_objects_ok": len(finals) == 4, + "staging_left": info["attachment"]["staging_key"] is not None, + "job_status": info["jobs"][0]["status"], + "attempt": info["jobs"][0]["attempt"], + } + + +def png(width, height, rgba=False, noise=False, color=(40, 120, 200), seed=1): + """PNG стандартной библиотекой: шум (несжимаемый) или заливка.""" + channels = 4 if rgba else 3 + rng = random.Random(seed) + rows = [] + if noise: + for _ in range(height): + rows.append(b"\x00" + bytes(rng.getrandbits(8) for _ in range(width * channels))) + else: + px = bytes(color[:3]) + (b"\x80" if rgba else b"") + row = b"\x00" + px * width + rows = [row] * height + raw = b"".join(rows) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ihdr = struct.pack(">IIBBBBB", width, height, 8, 6 if rgba else 2, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 6)) + + chunk(b"IEND", b"")) + + +def noise_png_fast(width, height, seed=1): + """Несжимаемый PNG через os.urandom (быстро для больших размеров).""" + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"") diff --git a/student-tests/lab7_api_security.py b/student-tests/lab7_api_security.py new file mode 100644 index 0000000..a3bbddc --- /dev/null +++ b/student-tests/lab7_api_security.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Lab7: отрицательные проверки внешнего API через HTTPS-ingress (с хоста). + + python3 student-tests/lab7_api_security.py --ca-file secrets/pki/public-ca.crt \ + --output evidence/lab7-api-security.json + +Покрывает: внешний TLS (CA, hostname, внутренний CA не подходит, TLS<1.2), посторонний пользователь +(metadata/content/история/прикрепление чужого файла), поддельный X-User-Id, Basic на бизнес-endpoint, +logout, expiry сессии, одинаковый 401 без раскрытия существования логина, rate limit входа +(429 + Retry-After), лимиты размера (JSON 1 MiB, upload 10 MiB), заголовки ответа (alias реплики, +нет Server/hostname), Argon2id в БД (параметры, соль) и rehash старого scrypt-хеша. + +Для expiry/rehash/формата хеша тест обращается к PostgreSQL как ОПЕРАТОР +(`docker compose exec db psql` через локальный сокет, peer) — только к строкам своих тестовых +пользователей. Токены и пароли не печатаются и не сохраняются. +""" +import argparse +import base64 +import hashlib +import json +import os +import socket +import ssl +import subprocess +import sys +import time +import uuid + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +from client import Client, Failure # noqa: E402 + +RESULTS = [] + + +def ok(name, cond, detail=""): + RESULTS.append({"check": name, "pass": bool(cond), "detail": detail}) + print("%s %s %s" % ("PASS" if cond else "FAIL", name, detail)) + + +def psql(sql): + out = subprocess.run(["docker", "compose", "exec", "-T", "db", "su-exec", "postgres", "psql", "-qAt", + "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, text=True, timeout=30) + if out.returncode != 0: + raise RuntimeError("psql failed: %s" % out.stderr.strip()[:200]) + return out.stdout.strip() + + +def tls_probe(host, port, cafile, server_hostname, max_version=None): + c = ssl.create_default_context(cafile=cafile) + if max_version: + c.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + import warnings + warnings.simplefilter("ignore", DeprecationWarning) + c.maximum_version = max_version + c.set_ciphers("ALL:@SECLEVEL=0") + try: + with socket.create_connection((host, port), timeout=5) as raw, \ + c.wrap_socket(raw, server_hostname=server_hostname) as tls: + return "ok", tls.version() + except ssl.SSLCertVerificationError as exc: + return "cert_error", exc.verify_message + except ssl.SSLError as exc: + return "tls_error", exc.reason + except (socket.timeout, TimeoutError): + return "timeout", "" + + +def basic(username, password): + return "Basic " + base64.b64encode(("%s:%s" % (username, password)).encode()).decode() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) + ap.add_argument("--ca-file", default=os.environ.get("CA_FILE", os.path.join(ROOT, "secrets/pki/public-ca.crt"))) + ap.add_argument("--output") + ap.add_argument("--skip-rate-limit", action="store_true") + args = ap.parse_args() + ca_internal = os.path.join(ROOT, "secrets/pki/internal-ca.crt") + c = Client(args.base_url, args.ca_file) + _call = c.call + + def call(*a, **kw): # публичный клиент разбирает JSON только для ошибок/схем + r = _call(*a, **kw) + if r["json"] is None and r["bytes"] and r["headers"].get_content_type() == "application/json": + r["json"] = json.loads(r["bytes"]) + return r + c.call = call + host = "localhost" + port = int(args.base_url.rsplit(":", 1)[1].split("/")[0]) + + # ---------------- внешний TLS + st = tls_probe(host, port, args.ca_file, "localhost") + ok("tls.trusted_ca_and_hostname", st[0] == "ok", st[1]) + st = tls_probe("127.0.0.1", port, args.ca_file, "evil.example") + ok("tls.wrong_hostname_rejected", st[0] == "cert_error" and "Hostname mismatch" in (st[1] or ""), st[1]) + st = tls_probe(host, port, ca_internal, "localhost") + ok("tls.internal_ca_does_not_validate_ingress", st[0] == "cert_error", st[1]) + st = tls_probe(host, port, args.ca_file, "localhost", max_version=ssl.TLSVersion.TLSv1_1) + ok("tls.tls11_refused", st[0] in ("tls_error",), "%s %s" % st) + + # ---------------- пользователи, чат, вложение + # Rate limit входа общий на LB по IP: если перед тестом шли входы (smoke и т. п.), ждём окно 10 с. + time.sleep(11) + alice, bob, eve = c.user("alice"), c.user("bob"), c.user("eve") + chat = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "lab7 security", "member_ids": [alice["id"], bob["id"]]})["json"] + png = open(os.path.join(ROOT, "tests/fixtures/sample.png"), "rb").read() + att = c.call("POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 202, actor=alice, + raw=png, headers={"Content-Type": "image/png"})["json"] + msg = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 201, actor=alice, + body={"text": "секрет чата"}, headers={"Idempotency-Key": uuid.uuid4().hex})["json"] + deadline = time.time() + 60 + while time.time() < deadline: + meta = c.call("GET", "/api/v1/attachments/%s" % att["id"], 200, actor=alice)["json"] + if meta["status"] in ("ready", "failed"): + break + time.sleep(0.5) + ok("setup.attachment_ready", meta["status"] == "ready", meta["status"]) + + # ---------------- посторонний + for name, path in (("chat", "/api/v1/chats/%s" % chat["id"]), + ("history", "/api/v1/chats/%s/messages" % chat["id"]), + ("attachment_metadata", "/api/v1/attachments/%s" % att["id"]), + ("attachment_content", "/api/v1/attachments/%s/content" % att["id"]), + ("attachment_thumbnail", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"])): + r = c.call("GET", path, [404, 200, 403, 401], actor=eve) + ok("outsider.%s_404" % name, r["status"] == 404, str(r["status"])) + eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve, body={"title": "eve", "member_ids": [eve["id"]]})["json"] + r = c.call("POST", "/api/v1/chats/%s/messages" % eve_chat["id"], [404, 201, 400, 409], actor=eve, + body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + ok("outsider.attach_foreign_file_404", r["status"] == 404, str(r["status"])) + r = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], [404, 201], actor=eve, + body={"text": "x"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + ok("outsider.post_into_foreign_chat_404", r["status"] == 404, str(r["status"])) + r = c.call("GET", "/api/v1/chats", 200, actor=eve)["json"] + ok("outsider.chat_list_hides_foreign", all(x["id"] != chat["id"] for x in r["items"]), "") + + # ---------------- подмена identity + r = c.call("GET", "/api/v1/users/me", 200, actor=eve, headers={"X-User-Id": alice["id"]})["json"] + ok("spoof.x_user_id_ignored", r["id"] == eve["id"], "me=%s" % ("eve" if r["id"] == eve["id"] else "OTHER")) + r = c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + ok("spoof.x_user_id_alone_401", r["status"] == 401, "") + reg = alice["registration"] + r = c.call("GET", "/api/v1/chats", 401, headers={"Authorization": basic(reg["username"], reg["password"])}) + ok("auth.basic_on_business_endpoint_401", r["status"] == 401, "") + r = c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "A" * 43}) + ok("auth.unknown_token_401", r["status"] == 401, r["headers"].get("WWW-Authenticate", "")) + + # ---------------- одинаковый отказ: неизвестный логин vs неверный пароль + r1 = c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(reg["username"], "wrong-password-123")}) + r2 = c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("no_such_" + uuid.uuid4().hex[:8], "wrong-password-123")}) + same = (r1["json"]["error"]["code"], r1["json"]["error"]["message"], r1["headers"].get("WWW-Authenticate")) == \ + (r2["json"]["error"]["code"], r2["json"]["error"]["message"], r2["headers"].get("WWW-Authenticate")) + ok("auth.same_401_unknown_user_and_wrong_password", same, "") + body = r1["bytes"].decode("utf-8", "replace") + ok("auth.error_has_no_credentials_or_internals", + not any(x in body.lower() for x in ("traceback", "argon", "select ", "password_hash", reg["password"].lower())), "") + + # ---------------- logout + bob_token = bob["token"] + r = c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob) + ok("logout.204", r["status"] == 204, "") + r = c.call("GET", "/api/v1/users/me", 401, actor={"token": bob_token}) + ok("logout.token_revoked_401", r["status"] == 401, "") + r = c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor={"token": bob_token}) + ok("logout.repeat_401", r["status"] == 401, "") + + # ---------------- expiry (оператор сдвигает expires_at ТОЛЬКО своей тестовой сессии) + carol = c.user("carol") + th = hashlib.sha256(carol["token"].encode()).hexdigest() + c.call("GET", "/api/v1/users/me", 200, actor=carol) + n = psql("UPDATE sessions SET expires_at = now() - interval '1 second' WHERE token_hash = decode('%s','hex') RETURNING 1" % th) + r = c.call("GET", "/api/v1/users/me", 401, actor=carol) + ok("expiry.expired_session_401", r["status"] == 401 and n == "1", "rows=%s" % n) + + # ---------------- Argon2id в БД + h = psql("SELECT password_hash FROM users WHERE id = '%s'" % alice["id"]) + parts = h.split("$") + ok("argon2.stored_format", h.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and len(parts) == 6, + "$".join(parts[:4]) + "$$") + h2 = psql("SELECT password_hash FROM users WHERE id = '%s'" % eve["id"]) + ok("argon2.unique_salt", h.split("$")[4] != h2.split("$")[4], "") + + # ---------------- rehash: старый scrypt (lab2–6) -> Argon2id при успешном входе + dave = c.user("dave") + pw = dave["registration"]["password"] + salt = os.urandom(16) + dg = hashlib.scrypt(pw.encode(), salt=salt, n=2 ** 15, r=8, p=1, maxmem=128 * 1024 * 1024, dklen=32) + legacy = "scrypt$15$8$1$%s$%s" % (base64.b64encode(salt).decode(), base64.b64encode(dg).decode()) + psql("UPDATE users SET password_hash = '%s' WHERE id = '%s'" % (legacy, dave["id"])) + r = c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": basic(dave["registration"]["username"], pw)}) + after = psql("SELECT password_hash FROM users WHERE id = '%s'" % dave["id"]) + ok("argon2.rehash_legacy_scrypt_on_login", r["status"] == 201 and after.startswith("$argon2id$"), after[:30]) + + # ---------------- лимиты размера + big = {"username": "x" * 10, "display_name": "y" * (1024 * 1024 + 10), "password": "z" * 20} + r = c.call("POST", "/api/v1/users", 413, body=big) + ok("limits.json_body_over_1mib_413", r["status"] == 413, r["json"]["error"]["code"]) + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=b.png" % chat["id"], 413, actor=alice, + raw=b"\x89PNG" + b"\0" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"}) + ok("limits.upload_over_10mib_413", r["status"] == 413, r["json"]["error"]["code"]) + + # ---------------- заголовки ответа + r = c.call("GET", "/health/ready", 200) + iid = r["headers"].get("X-Instance-Id", "") + ok("headers.instance_alias_random", iid.startswith("r-") and len(iid) == 12, iid) + ok("headers.no_server_banner", r["headers"].get("Server") is None, str(r["headers"].get("Server"))) + ok("headers.hsts", "max-age" in (r["headers"].get("Strict-Transport-Security") or ""), "") + + # ---------------- rate limit входа (общий на LB для всех реплик) + if not args.skip_rate_limit: + time.sleep(11) # окно http_req_rate(10s) от предыдущих входов теста + import urllib.error + import urllib.request + ctx = ssl.create_default_context(cafile=args.ca_file) + statuses, retry_after, err, rid_ok = [], None, None, False + for i in range(30): + # 429 формирует сам LB (stick-table): X-Instance-Id у такого ответа нет — это допустимо + # контрактом, поэтому здесь сырые запросы, а не публичный Client. + req = urllib.request.Request(args.base_url + "/api/v1/auth/sessions", method="POST", headers={ + "Authorization": basic("rl_" + uuid.uuid4().hex[:8], "wrong-password-123")}) + try: + with urllib.request.urlopen(req, timeout=10, context=ctx) as resp: + code, hdrs, data = resp.status, resp.headers, resp.read() + except urllib.error.HTTPError as exc: + code, hdrs, data = exc.code, exc.headers, exc.read() + statuses.append(code) + if code == 429 and retry_after is None: + retry_after = hdrs.get("Retry-After") + err = json.loads(data)["error"]["code"] + rid_ok = json.loads(data)["error"]["request_id"] == hdrs.get("X-Request-Id") + first = statuses.index(429) + 1 if 429 in statuses else None + ok("ratelimit.login_429_with_retry_after", first is not None and retry_after and err == "rate_limited" and rid_ok, + "first 429 at attempt %s, Retry-After=%s, code=%s, request_id matches header=%s" % (first, retry_after, err, rid_ok)) + time.sleep(11) + r = c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": basic(reg["username"], reg["password"])}) + ok("ratelimit.recovers_after_window", r["status"] == 201, "") + + failed = [x for x in RESULTS if not x["pass"]] + if args.output: + with open(args.output, "w") as f: + json.dump({"checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "base_url": args.base_url, + "total": len(RESULTS), "failed": len(failed), "results": RESULTS}, f, ensure_ascii=False, indent=1) + print("%s: %d checks, %d failed" % ("PASS" if not failed else "FAIL", len(RESULTS), len(failed))) + return 1 if failed else 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + sys.exit(1) diff --git a/student-tests/lab7_hardening.py b/student-tests/lab7_hardening.py new file mode 100644 index 0000000..dd5195e --- /dev/null +++ b/student-tests/lab7_hardening.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Lab7: hardening контейнеров и отсутствие утечек секретов (Git, образ, env, телеметрия). + + python3 student-tests/lab7_hardening.py --output evidence/lab7-hardening.json + +1. docker inspect каждого запущенного контейнера проекта: не privileged, не host network/pid/ipc, + нет Docker socket, no-new-privileges, лимит памяти/CPU; для процессов приложения — uid 10001, + CapDrop ALL, read-only rootfs; для остальных — допустимые исключения перечислены явно. +2. Опубликованные на хост порты: только 127.0.0.1:8443 (lb) и 127.0.0.1:3000 (grafana). +3. Env контейнеров: нет BAO_TOKEN/VAULT_TOKEN/SecretID и значений паролей/ключей (только *_FILE пути). +4. Образ приложения: нет *.key/*.pem/секретов/каталога secrets; история слоёв без секретов. +5. Репозиторий: scripts/scan_secrets.py (значения из secrets/ и каталога оператора). +6. Телеметрия: свежий Bearer-токен и пароль тестового пользователя не находятся в Loki (логи) и + Tempo (атрибуты spans) — запросы через Grafana (HTTPS) после реальных запросов с ними. +""" +import argparse +import json +import os +import subprocess +import sys +import time +import uuid + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "student-tests")) +RESULTS = [] +APP_SERVICES = {"api", "worker", "relay", "janitor"} +# Документированные исключения (stateful-сервисы с entrypoint-ами образов, см. REPORT.md): +CAP_EXCEPTIONS = {"db": {"CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"}, + "broker": {"CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"}, + "s3": {"CHOWN", "SETGID", "SETUID"}} +RW_EXCEPTIONS = {"db", "broker", "s3", "grafana"} + + +def ok(name, cond, detail=""): + RESULTS.append({"check": name, "pass": bool(cond), "detail": detail}) + print("%s %s %s" % ("PASS" if cond else "FAIL", name, detail)) + + +def sh(*cmd, timeout=60): + return subprocess.run(list(cmd), cwd=ROOT, capture_output=True, text=True, timeout=timeout) + + +def containers(): + ids = sh("docker", "compose", "ps", "-q").stdout.split() + return json.loads(sh("docker", "inspect", *ids).stdout) + + +def check_containers(): + allowed_ports = {("lb", "8443/tcp"): "127.0.0.1", ("grafana", "3000/tcp"): "127.0.0.1"} + for c in containers(): + svc = c["Config"]["Labels"].get("com.docker.compose.service") + hc, cfg = c["HostConfig"], c["Config"] + n = "%s(%s)" % (svc, c["Name"].lstrip("/").rsplit("-", 1)[-1]) + ok("%s.not_privileged" % n, not hc.get("Privileged")) + ok("%s.no_host_namespaces" % n, hc.get("NetworkMode") != "host" and hc.get("PidMode") != "host" + and hc.get("IpcMode") != "host", "%s/%s/%s" % (hc.get("NetworkMode"), hc.get("PidMode"), hc.get("IpcMode"))) + mounts = [m.get("Source", "") for m in c.get("Mounts", [])] + ok("%s.no_docker_socket" % n, not any("docker.sock" in m for m in mounts)) + ok("%s.no_new_privileges" % n, any("no-new-privileges" in x for x in (hc.get("SecurityOpt") or []))) + ok("%s.memory_cpu_limits" % n, (hc.get("Memory") or 0) > 0 and (hc.get("NanoCpus") or 0) > 0, + "mem=%dMiB cpu=%.2f" % ((hc.get("Memory") or 0) >> 20, (hc.get("NanoCpus") or 0) / 1e9)) + caps_drop = set(hc.get("CapDrop") or []) + caps_add = {x.replace("CAP_", "") for x in (hc.get("CapAdd") or [])} + ok("%s.cap_drop_all" % n, "ALL" in caps_drop, "add=%s" % ",".join(sorted(caps_add)) if caps_add else "") + ok("%s.cap_add_only_documented" % n, caps_add <= CAP_EXCEPTIONS.get(svc, set()), ",".join(sorted(caps_add))) + if svc in APP_SERVICES: + ok("%s.non_root_uid" % n, cfg.get("User") == "10001:10001", cfg.get("User")) + ok("%s.read_only_rootfs" % n, hc.get("ReadonlyRootfs") is True) + elif svc not in RW_EXCEPTIONS: + ok("%s.read_only_rootfs" % n, hc.get("ReadonlyRootfs") is True) + for port, binds in (hc.get("PortBindings") or {}).items(): + for b in binds or []: + ok("%s.published_%s" % (n, port), allowed_ports.get((svc, port)) == b.get("HostIp"), + "%s:%s" % (b.get("HostIp"), b.get("HostPort"))) + env = cfg.get("Env") or [] + names = [e.split("=", 1)[0] for e in env] + ok("%s.no_vault_token_in_env" % n, not any(x in ("BAO_TOKEN", "VAULT_TOKEN") or "SECRET_ID" in x for x in names)) + leaky = [x for x in names if any(k in x for k in ("PASSWORD", "SECRET_ACCESS_KEY", "ACCESS_KEY_ID", "BROKER_URL")) + and not x.endswith("_FILE") and x not in ("PASSWORD_HASH_CONCURRENCY", "PASSWORD_HASH_WAIT_SECONDS")] + ok("%s.no_secret_values_in_env" % n, not leaky, ",".join(leaky)) + if svc in APP_SERVICES: + m = [m.get("Destination") for m in c.get("Mounts", [])] + ok("%s.only_own_tls_identity_mounted" % n, + sorted(x for x in m if x.startswith("/run/secrets")) == + ["/run/secrets/internal-ca.crt", "/run/secrets/service.crt", "/run/secrets/service.key"], ",".join(m)) + + +HEALTH_PROBE = ("import urllib.request,urllib.error,json\n" + "for m,p in (('GET','/health/ready'),('GET','/api/v1/chats'),('POST','/api/v1/users'),('GET','/api/v1/users/me')):\n" + " r=urllib.request.Request('http://127.0.0.1:8081'+p,method=m,data=b'{}' if m=='POST' else None,headers={'Content-Type':'application/json'})\n" + " try:\n" + " with urllib.request.urlopen(r,timeout=3) as x: print(m,p,x.status)\n" + " except urllib.error.HTTPError as e: print(m,p,e.code)\n") + + +def check_health_listener(): + """Loopback health listener API (plaintext 127.0.0.1:8081) не даёт бизнес-API даже изнутри контейнера.""" + for cid in sh("docker", "compose", "ps", "-q", "api").stdout.split(): + out = sh("docker", "exec", cid, "python", "-c", HEALTH_PROBE).stdout.split("\n") + codes = dict((" ".join(l.split()[:2]), l.split()[2]) for l in out if l.strip()) + ok("api(%s).health_listener_only_health" % cid[:6], + codes.get("GET /health/ready") == "200" and all(codes.get(k) == "404" for k in + ("GET /api/v1/chats", "POST /api/v1/users", "GET /api/v1/users/me")), + json.dumps(codes, ensure_ascii=False)) + + +def check_image(): + script = ("import os,re,sys\n" + "bad=[]\n" + "for top in ('/app','/home','/root','/etc','/run','/tmp'):\n" + " for d,_,fs in os.walk(top):\n" + " for f in fs:\n" + " p=os.path.join(d,f)\n" + " if p.startswith('/etc/ssl') or '/site-packages/' in p: continue\n" + " if re.search(r'\\.(key|pem|p12|pfx)$|init\\.json$|_password$|secret_key$|^\\.env$',f): bad.append(p)\n" + "print(len(bad)); print('\\n'.join(bad[:20]))\n" + "print('secrets_dir' if os.path.exists('/app/secrets') else 'no_secrets_dir')\n") + out = sh("docker", "run", "--rm", "--network", "none", "--entrypoint", "python", "mtusi-messenger:lab7", "-c", script) + lines = out.stdout.split("\n") + ok("image.no_key_or_secret_files", lines and lines[0] == "0", " ".join(lines[1:4])) + ok("image.no_secrets_dir", "no_secrets_dir" in out.stdout) + hist = sh("docker", "history", "--no-trunc", "--format", "{{.CreatedBy}}", "mtusi-messenger:lab7").stdout + values = [] + for f in ("postgres_password", "db_api_password", "s3_api_secret_key", "broker_worker_password"): + values.append(open(os.path.join(ROOT, "secrets", f)).read().strip()) + ok("image.history_without_secret_values", not any(v in hist for v in values)) + user = sh("docker", "image", "inspect", "-f", "{{.Config.User}}", "mtusi-messenger:lab7").stdout.strip() + ok("image.default_user_non_root", user == "10001:10001", user) + + +def check_repo(): + out = sh(sys.executable, "scripts/scan_secrets.py") + ok("repo.scan_secrets_clean", out.returncode == 0, out.stdout.strip()) + + +def check_telemetry(): + from client import Client + import obslib + c = Client(os.environ.get("BASE_URL", "https://localhost:8443"), os.environ.get("CA_FILE", os.path.join(ROOT, "secrets/pki/public-ca.crt"))) + marker = "leak_" + uuid.uuid4().hex[:10] + actor = c.user(marker) + token, password = actor["token"], actor["registration"]["password"] + start = time.time() - 5 + c.call("GET", "/api/v1/users/me", 200, actor=actor) + c.call("GET", "/api/v1/chats", 200, actor=actor) + c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + token + "x"}) + time.sleep(20) # батчи SDK (1 с) + Collector (1 с) + Loki ingest + end = time.time() + 5 + seen = obslib.loki_query('{service_name="messenger-api"} |= "%s"' % actor["user"]["username"], start, end, limit=20) + # Положительный контроль: поиск в Loki реально работает (иначе «0 утечек» ничего не значит). + seen = seen or obslib.loki_query('{service_name="messenger-api"} |= "/api/v1/users/me"', start, end, limit=20) + ok("telemetry.loki_search_positive_control", len(seen) > 0, "matching lines: %d" % len(seen)) + for label, value in (("bearer_token", token), ("password", password), ("token_prefix", token[:20])): + hits = obslib.loki_query('{service_name=~"messenger-.+"} |= "%s"' % value, start, end, limit=5) + ok("telemetry.loki_no_%s" % label, len(hits) == 0, "hits=%d" % len(hits)) + traces = obslib.tempo_search('{ .messenger.request_id != "" }', start, end, limit=50) or {} + found = 0 + for t in (traces.get("traces") or [])[:20]: + full = obslib.tempo_trace(t["traceID"]) + if full and (token in json.dumps(full) or password in json.dumps(full)): + found += 1 + ok("telemetry.tempo_no_token_or_password", found == 0, "checked %d traces" % min(20, len(traces.get("traces") or []))) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--output") + args = ap.parse_args() + for fn in (check_containers, check_health_listener, check_image, check_repo, check_telemetry): + try: + fn() + except Exception as exc: + ok(fn.__name__ + ".crashed", False, repr(exc)[:200]) + failed = [x for x in RESULTS if not x["pass"]] + if args.output: + with open(args.output, "w") as f: + json.dump({"checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "total": len(RESULTS), + "failed": len(failed), "results": RESULTS}, f, ensure_ascii=False, indent=1) + print("%s: %d checks, %d failed" % ("PASS" if not failed else "FAIL", len(RESULTS), len(failed))) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/lab7_openbao.py b/student-tests/lab7_openbao.py new file mode 100644 index 0000000..47a1aaa --- /dev/null +++ b/student-tests/lab7_openbao.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Lab7: OpenBao — non-dev, persistent, scoped policies, отсутствие root credential, sealed-сценарий. + + python3 student-tests/lab7_openbao.py --ca-file secrets/pki/public-ca.crt --output evidence/lab7-openbao.json + +1. `bao status` (внутри контейнера, без токена): initialized, unsealed, storage raft, не dev. +2. src/selfcheck.py ВНУТРИ api/worker/relay/janitor (их собственная cert-identity): свой путь 200; + чужой сервис, admin-путь migrate, list metadata, запись секрета/policy, создание токена, seal, + список ролей — 403; TTL токена <= 900 с, renewable; после revoke-self токен бесполезен. +3. Worker получил СВОЙ ключ S3, а не admin/api (сравнение SHA-256-префиксов с secrets/). +4. Sealed: рестарт контейнера OpenBao -> sealed. Работающие API продолжают обслуживать запросы + (секреты в памяти, соединения с БД уже есть). Перезапущенная во время sealed API-реплика НЕ + становится healthy (ждёт OpenBao, не берёт значений по умолчанию). После unseal оператором — + реплика стартует, данные KV на месте (persistent storage). +""" +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +from client import Client # noqa: E402 + +RESULTS = [] + + +def ok(name, cond, detail=""): + RESULTS.append({"check": name, "pass": bool(cond), "detail": detail}) + print("%s %s %s" % ("PASS" if cond else "FAIL", name, detail)) + + +def sh(*cmd, timeout=120, check=False): + out = subprocess.run(list(cmd), cwd=ROOT, capture_output=True, text=True, timeout=timeout) + if check and out.returncode != 0: + raise RuntimeError("%s -> %d %s" % (" ".join(cmd[:4]), out.returncode, out.stderr[-300:])) + return out + + +def fp(path): + return hashlib.sha256(open(os.path.join(ROOT, "secrets", path)).read().strip().encode()).hexdigest()[:8] + + +def bao_status(): + out = sh("docker", "compose", "exec", "-T", "openbao", "bao", "status", "-format=json") + try: + return json.loads(out.stdout) + except ValueError: + return {} + + +def health(cid): + return sh("docker", "inspect", "-f", "{{.State.Health.Status}}", cid).stdout.strip() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) + ap.add_argument("--ca-file", default=os.environ.get("CA_FILE", os.path.join(ROOT, "secrets/pki/public-ca.crt"))) + ap.add_argument("--output") + ap.add_argument("--skip-sealed", action="store_true") + args = ap.parse_args() + + st = bao_status() + ok("status.initialized_unsealed", st.get("initialized") is True and st.get("sealed") is False) + ok("status.storage_raft_persistent", st.get("storage_type") == "raft", st.get("storage_type")) + ok("status.shares_threshold", (st.get("n") or st.get("t") and True) and st.get("t") == 2 and st.get("n") == 3, + "t=%s n=%s" % (st.get("t"), st.get("n"))) + cmd = sh("docker", "inspect", "-f", "{{json .Config.Entrypoint}} {{json .Config.Cmd}}", + sh("docker", "compose", "ps", "-q", "openbao").stdout.strip()).stdout + ok("status.not_dev_mode", "-dev" not in cmd and "server" in cmd, cmd.strip()) + + expect_fields = {"api": {"POSTGRES_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"}, + "worker": {"POSTGRES_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "BROKER_URL"}, + "relay": {"POSTGRES_PASSWORD", "BROKER_URL"}, + "janitor": {"POSTGRES_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"}} + reports = {} + for svc in ("api", "worker", "relay", "janitor"): + out = sh("docker", "compose", "exec", "-T", svc, "python", "/app/src/selfcheck.py", check=True) + rep = json.loads(out.stdout.strip().splitlines()[-1]) + reports[svc] = rep + b = rep["openbao"] + ok("%s.login_policies" % svc, b["login"]["policies"] == ["default", "messenger-%s" % svc], ",".join(b["login"]["policies"])) + ok("%s.token_ttl_bounded" % svc, 0 < b["login"]["ttl"] <= 900 and b["login"]["renewable"], "ttl=%s" % b["login"]["ttl"]) + ok("%s.read_own_path_200" % svc, b["read_own"] == 200) + ok("%s.own_fields_exact" % svc, set(b["own_fields"]) == expect_fields[svc], ",".join(b["own_fields"])) + for probe in ("read_other_service", "read_migrate_admin", "list_metadata", "write_own_secret", "write_policy", + "create_token", "seal", "list_auth_roles"): + ok("%s.deny_%s" % (svc, probe), b[probe] == 403, str(b[probe])) + ok("%s.renew_self" % svc, b["renew_self"] == 200) + ok("%s.revoked_token_denied" % svc, b["read_after_revoke"] == 403, str(b["read_after_revoke"])) + ok("%s.no_token_in_env" % svc, not rep["env_has_token"], ",".join(rep["env_has_token"])) + ok("%s.only_tls_identity_mounted" % svc, rep["mounted_secrets"] == ["internal-ca.crt", "service.crt", "service.key"], + ",".join(rep["mounted_secrets"])) + ok("%s.db_role_not_privileged" % svc, not any(rep["db"]["role_flags"].values()) and rep["db"]["ssl"]["ssl"], + "%s %s" % (rep["db"]["user"], rep["db"]["ssl"]["version"])) + w = reports["worker"]["openbao"]["own_fingerprints"]["S3_ACCESS_KEY_ID"] + ok("worker.has_own_s3_key", w == fp("s3_worker_access_key")) + ok("worker.no_admin_or_api_s3_key", w not in (fp("s3_admin_access_key"), fp("s3_api_access_key"))) + ok("worker.db_no_users_sessions_messages", all(reports["worker"]["db"][k] == "denied" + for k in ("select_users", "select_sessions", "select_messages"))) + ok("api.db_no_ddl_truncate", all(reports["api"]["db"][k] == "denied" + for k in ("create_table", "drop_users", "truncate_outbox", "update_chat_title"))) + + if not args.skip_sealed: + c = Client(args.base_url, args.ca_file) + sh("docker", "compose", "restart", "openbao", check=True) + time.sleep(3) + st = bao_status() + ok("sealed.after_restart_sealed", st.get("sealed") is True) + u = c.user("sealed") # вход + регистрация: секреты уже в памяти API + r = c.call("GET", "/api/v1/users/me", 200, actor=u) + ok("sealed.running_api_keeps_serving", r["status"] == 200) + api_ids = sh("docker", "compose", "ps", "-q", "api").stdout.split() + target = api_ids[0] + t0 = time.time() + sh("docker", "restart", "-t", "20", target, check=True) + time.sleep(25) + hs = health(target) + logs = sh("docker", "logs", "--since", "40s", target).stdout + sh("docker", "logs", "--since", "40s", target).stderr + ok("sealed.restarted_replica_not_healthy", hs != "healthy", hs) + ok("sealed.restarted_replica_waits_for_openbao", "vault.unavailable" in logs or "openbao not ready" in logs) + r = c.call("GET", "/health/ready", 200) + ok("sealed.lb_serves_from_other_replica", r["status"] == 200, r["headers"].get("X-Instance-Id")) + sh("./scripts/bao-unseal.sh", check=True) + t_unseal = time.time() + while time.time() - t_unseal < 90 and health(target) != "healthy": + time.sleep(1) + ok("sealed.replica_healthy_after_unseal", health(target) == "healthy", + "%.1fs after unseal (total %.0fs)" % (time.time() - t_unseal, time.time() - t0)) + out = sh("docker", "exec", target, "python", "/app/src/selfcheck.py", check=True) + rep = json.loads(out.stdout.strip().splitlines()[-1]) + ok("sealed.kv_persisted_after_restart", rep["openbao"]["read_own"] == 200) + + failed = [x for x in RESULTS if not x["pass"]] + if args.output: + with open(args.output, "w") as f: + json.dump({"checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "total": len(RESULTS), + "failed": len(failed), "results": RESULTS}, f, ensure_ascii=False, indent=1) + print("%s: %d checks, %d failed" % ("PASS" if not failed else "FAIL", len(RESULTS), len(failed))) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/lab7_tls_matrix.py b/student-tests/lab7_tls_matrix.py new file mode 100644 index 0000000..2bc7e33 --- /dev/null +++ b/student-tests/lab7_tls_matrix.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Lab7: положительные и отрицательные проверки TLS/mTLS по contracts/trust-matrix.md. + +Запуск ВНУТРИ стенда (сервис checks видит сети apinet/backend/telemetry/vault): + docker compose --profile tools run --rm -T checks /app/student-tests/lab7_tls_matrix.py \ + --output /app/evidence/lab7-tls-matrix.json + +Для каждой связи различаются причины отказа: ошибка проверки сертификата, отказ handshake +сервером (нет/чужой client cert), закрытие соединения после handshake (запрещённая identity), +отказ plaintext. Сетевой timeout НЕ засчитывается как правильный отказ (FAIL). + +Identities (Docker secrets сервиса checks, только для тестов): + lb.crt/key — разрешённая identity LB (internal-ca) -> положительный путь к API + intruder.crt/key — internal-ca, SAN intruder (доверенный CA, чужая роль) -> должен быть отказ + rogue-lb.crt/key — rogue CA, SAN lb (правильное имя, чужой CA) -> должен быть отказ +""" +import argparse +import json +import socket +import ssl +import sys +import time +import urllib.error +import urllib.request + +S = "/run/secrets/" +CA = S + "internal-ca.crt" +PUBLIC_CA = S + "public-ca.crt" +ROGUE_CA = S + "rogue-ca.crt" +ID = {"lb": (S + "lb.crt", S + "lb.key"), "intruder": (S + "intruder.crt", S + "intruder.key"), + "rogue-lb": (S + "rogue-lb.crt", S + "rogue-lb.key"), None: None} +RESULTS = [] + + +def ctx(identity=None, cafile=CA, hostname_check=True): + c = ssl.create_default_context(cafile=cafile) + c.check_hostname = hostname_check + c.minimum_version = ssl.TLSVersion.TLSv1_2 + if identity: + c.load_cert_chain(*ID[identity]) + return c + + +def classify(exc): + if isinstance(exc, ssl.SSLCertVerificationError): + return "server_cert_rejected_by_client:" + (exc.verify_message or "") + if isinstance(exc, (socket.timeout, TimeoutError)): + return "TIMEOUT" + if isinstance(exc, ConnectionRefusedError): + return "connection_refused" + if isinstance(exc, ssl.SSLError): + return "tls_alert:" + (exc.reason or type(exc).__name__) + if isinstance(exc, (ConnectionResetError, BrokenPipeError)): + return "reset_by_server" + if isinstance(exc, socket.gaierror): + return "dns:" + str(exc) + return type(exc).__name__ + ":" + str(exc)[:80] + + +def https_probe(host, port, path, identity=None, server_hostname=None, cafile=CA, ip=None): + """TLS + HTTP/1.1 GET. -> ("ok", status) | ("fail", reason).""" + target = ip or host + try: + with socket.create_connection((target, port), timeout=4) as raw: + with ctx(identity, cafile).wrap_socket(raw, server_hostname=server_hostname or host) as tls: + req = "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n" % (path, host) + tls.sendall(req.encode()) + data = b"" + while True: + chunk = tls.recv(65536) + if not chunk: + break + data += chunk + if len(data) > 1 << 20: + break + if not data: + return "fail", "closed_after_handshake_no_http_response" + status = int(data.split(b" ", 2)[1]) + return "ok", status + except Exception as exc: + return "fail", classify(exc) + + +def plaintext_probe(host, port, payload=b"GET / HTTP/1.1\r\nHost: x\r\n\r\n"): + """Plaintext к TLS/закрытому порту: refused или не-HTTP ответ/закрытие = отказ plaintext.""" + try: + with socket.create_connection((host, port), timeout=4) as s: + s.sendall(payload) + s.settimeout(4) + data = s.recv(200) + if data.startswith(b"HTTP/") and b" 400" in data[:20] and b"HTTPS" in data: + return "fail", "http_400_plain_http_sent_to_https_port" + if data.startswith(b"HTTP/1.") and (b" 200" in data[:16] or b" 401" in data[:16] or b" 404" in data[:16]): + return "ok", data[:30].decode("latin1") + if not data: + return "fail", "closed_without_plaintext_response" + return "fail", "non_http_or_tls_alert_bytes:%s" % data[:8].hex() + except Exception as exc: + return "fail", classify(exc) + + +def record(link, case, outcome, expect, note=""): + status, detail = outcome + passed = (status == expect) and not (status == "fail" and str(detail) == "TIMEOUT") + RESULTS.append({"link": link, "case": case, "expect": expect, "observed": status, "detail": detail, + "pass": passed, "note": note}) + print("%s %-28s %-48s expect=%-4s got=%s %s" % ("PASS" if passed else "FAIL", link, case, expect, status, detail)) + + +def api_ips(): + return sorted({ai[4][0] for ai in socket.getaddrinfo("api", 8443, proto=socket.IPPROTO_TCP)}) + + +def check_api(): + ips = api_ips() + print("API replicas:", len(ips)) + RESULTS.append({"link": "LB->API", "case": "replicas_resolved", "pass": len(ips) >= 2, "detail": len(ips)}) + for i, ip in enumerate(ips): + link = "LB->API[%d]" % i + record(link, "no client cert", https_probe("api", 8443, "/health/ready", None, "api", ip=ip), "fail") + record(link, "client cert of rogue CA, SAN lb", https_probe("api", 8443, "/health/ready", "rogue-lb", "api", ip=ip), "fail") + record(link, "trusted CA, identity intruder", https_probe("api", 8443, "/health/ready", "intruder", "api", ip=ip), "fail") + record(link, "plaintext HTTP to 8443", plaintext_probe(ip, 8443), "fail") + record(link, "health listener 8081 from network", plaintext_probe(ip, 8081), "fail") + record(link, "server cert, wrong expected hostname", https_probe("api", 8443, "/health/ready", "lb", "db", ip=ip), "fail") + ok = https_probe("api", 8443, "/health/ready", "lb", "api", ip=ip) + record(link, "identity lb -> /health/ready", ok, "ok", "status=%s" % (ok[1],)) + biz = https_probe("api", 8443, "/api/v1/users/me", "lb", "api", ip=ip) + record(link, "identity lb, no Bearer -> business API", biz, "ok", + "mTLS не заменяет сессию: ожидается 401, получено %s" % (biz[1],)) + if biz[0] == "ok" and biz[1] != 401: + RESULTS[-1]["pass"] = False + + +def check_pg(): + import psycopg + base = dict(host="db", dbname="messenger", user="api_svc", password="wrong-password-for-probe", connect_timeout=4) + + def attempt(**kw): + try: + psycopg.connect(**{**base, **kw}).close() + return "ok", "connected" + except psycopg.OperationalError as exc: + msg = str(exc).lower() + if "password authentication failed" in msg: + return "ok", "tls_ok_then_password_rejected" + if "no pg_hba.conf entry" in msg and "no encryption" in msg: + return "fail", "pg_hba_reject_plaintext" + if "certificate verify failed" in msg or "does not match host name" in msg or "server certificate" in msg: + return "fail", "server_cert_rejected_by_client" + if "timeout" in msg: + return "fail", "TIMEOUT" + return "fail", msg.strip()[:120] + + record("API/worker->PostgreSQL", "sslmode=disable (plaintext)", attempt(sslmode="disable"), "fail") + record("API/worker->PostgreSQL", "verify-full, rogue CA as trust", attempt(sslmode="verify-full", sslrootcert=ROGUE_CA), "fail") + ip = socket.gethostbyname("db") + record("API/worker->PostgreSQL", "verify-full by IP (SAN mismatch)", attempt(host=ip, sslmode="verify-full", sslrootcert=CA), "fail") + record("API/worker->PostgreSQL", "verify-full, internal CA, wrong pwd", attempt(sslmode="verify-full", sslrootcert=CA), "ok", + "TLS проверен; дальше SCRAM отклоняет неверный пароль") + + +def check_s3(): + record("API/worker->S3", "plaintext HTTP", plaintext_probe("s3", 8333), "fail") + record("API/worker->S3", "HTTPS, rogue CA as trust", https_probe("s3", 8333, "/", cafile=ROGUE_CA), "fail") + anon = https_probe("s3", 8333, "/messenger-attachments/") + record("API/worker->S3", "HTTPS verified, anonymous list", anon, "ok", "TLS ok; анонимно: %s (ожидается 403)" % (anon[1],)) + if anon[0] == "ok" and anon[1] not in (401, 403): + RESULTS[-1]["pass"] = False + + +def amqp_tls(identity=None, cafile=CA): + try: + with socket.create_connection(("broker", 5671), timeout=4) as raw: + with ctx(identity, cafile).wrap_socket(raw, server_hostname="broker") as tls: + tls.sendall(b"AMQP\x00\x00\x09\x01") + data = tls.recv(16) + return ("ok", "amqp_connection_start") if data[:1] == b"\x01" else ("fail", data[:8].hex()) + except Exception as exc: + return "fail", classify(exc) + + +def check_broker(): + record("API/worker->broker", "plaintext AMQP 5672", plaintext_probe("broker", 5672, b"AMQP\x00\x00\x09\x01"), "fail") + record("API/worker->broker", "AMQPS, rogue CA as trust", amqp_tls(cafile=ROGUE_CA), "fail") + record("API/worker->broker", "AMQPS, client cert of rogue CA", amqp_tls(identity="rogue-lb"), "fail", + "verify_peer: предъявленный чужой сертификат отвергается") + record("API/worker->broker", "AMQPS verified (без cert), Connection.Start", amqp_tls(), "ok", + "далее аутентификация паролем + ACL vhost (проверка прав — broker_permissions)") + try: + import pika + params = pika.ConnectionParameters(host="broker", port=5671, virtual_host="messenger", + credentials=pika.PlainCredentials("worker", "wrong-password"), + ssl_options=pika.SSLOptions(ctx(), server_hostname="broker"), + connection_attempts=1, socket_timeout=4) + pika.BlockingConnection(params).close() + out = ("ok", "LOGIN_ACCEPTED") + except pika.exceptions.ProbableAuthenticationError: + out = ("fail", "auth_refused") + except Exception as exc: + out = ("fail", type(exc).__name__) + record("API/worker->broker", "AMQPS, user worker, wrong password", out, "fail") + record("Prometheus->broker metrics", "HTTPS 15691 without client cert", https_probe("broker", 15691, "/metrics"), "fail") + record("Prometheus->broker metrics", "plaintext 15692 from network", plaintext_probe("broker", 15692), "fail") + + +def check_telemetry(): + for host, port, link in (("otel-collector", 4318, "API/worker->Collector"), + ("prometheus", 9090, "Collector/Grafana->Prometheus"), + ("loki", 3100, "Collector/Grafana->Loki"), + ("tempo", 3200, "Grafana->Tempo"), + ("tempo", 4317, "Collector->Tempo OTLP gRPC"), + ("lb", 8404, "Prometheus->LB metrics")): + record(link, "plaintext", plaintext_probe(host, port), "fail") + record(link, "TLS without client cert", https_probe(host, port, "/", None), "fail") + record(link, "TLS, client cert of rogue CA", https_probe(host, port, "/", "rogue-lb"), "fail") + record(link, "TLS, wrong expected hostname", https_probe(host, port, "/", "intruder", server_hostname="api"), "fail") + for host, port in (("otel-collector", 4317), ("otel-collector", 8888), ("otel-collector", 13133), ("loki", 9095), ("tempo", 9096)): + record("internal-only port", "%s:%d from network" % (host, port), plaintext_probe(host, port), "fail") + # Уровень 3: Collector/backends проверяют CA, но не роль клиента -> intruder (internal-ca) принят. + got = https_probe("otel-collector", 4318, "/v1/logs", "intruder") + RESULTS.append({"link": "API/worker->Collector", "case": "trusted CA, foreign identity (intruder)", + "expect": "documented", "observed": got[0], "detail": got[1], "pass": True, + "note": "level 3: CA-only mTLS; разграничение identity — уровень 4 (см. trust-matrix)"}) + print("INFO API/worker->Collector intruder cert:", got) + + +def bao_login(identity): + c = ctx(identity) + req = urllib.request.Request("https://openbao:8200/v1/auth/cert/login", data=b'{"name":"api"}', method="POST", + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=4, context=c) as r: + return "ok", r.status + except urllib.error.HTTPError as exc: + return "fail", "http_%d" % exc.code + except Exception as exc: + return "fail", classify(getattr(exc, "reason", exc)) + + +def check_openbao(): + record("services->OpenBao", "plaintext HTTP", plaintext_probe("openbao", 8200), "fail") + record("services->OpenBao", "TLS, rogue CA as trust", https_probe("openbao", 8200, "/v1/sys/health", cafile=ROGUE_CA), "fail") + record("services->OpenBao", "TLS verified, /v1/sys/health", https_probe("openbao", 8200, "/v1/sys/health"), "ok") + record("services->OpenBao", "cert login as role api with intruder cert", bao_login("intruder"), "fail") + record("services->OpenBao", "cert login as role api with lb cert", bao_login("lb"), "fail", + "сертификат того же CA, но SAN не api -> роль не подходит") + record("services->OpenBao", "cert login without client cert", bao_login(None), "fail") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--output") + args = ap.parse_args() + started = time.time() + for fn in (check_api, check_pg, check_s3, check_broker, check_telemetry, check_openbao): + try: + fn() + except Exception as exc: + RESULTS.append({"link": fn.__name__, "case": "crashed", "pass": False, "detail": repr(exc)[:200]}) + print("FAIL", fn.__name__, repr(exc)[:200]) + failed = [r for r in RESULTS if not r.get("pass")] + summary = {"checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(started)), + "total": len(RESULTS), "failed": len(failed), "results": RESULTS} + if args.output: + with open(args.output, "w") as f: + json.dump(summary, f, ensure_ascii=False, indent=1) + print("%s: %d checks, %d failed" % ("PASS" if not failed else "FAIL", len(RESULTS), len(failed))) + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/student-tests/lease_takeover.py b/student-tests/lease_takeover.py new file mode 100644 index 0000000..59abfc0 --- /dev/null +++ b/student-tests/lease_takeover.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Уровень 4: зависший processing (lease) и fencing опоздавшего worker. + + python3 student-tests/lease_takeover.py --output evidence/lease-takeover.json + +Worker пересоздаются с WORKER_FAULT_INJECTION=1, JOB_LEASE_SECONDS=10, FAULT_SLOW_SECONDS=25. + 1. upload fault-slow.png: worker берёт задание и «долго работает» (продлевает аренду); + 2. пока аренда продлевается, второй worker задание не трогает (долгий != зависший); + 3. docker pause контейнера-владельца: процесс жив, но заморожен — аренду никто не продлевает; + 4. через ~lease reaper (relay) переводит задание в queued, второй worker берёт его -> ready; + 5. docker unpause: опоздавший worker обнаруживает потерю аренды и ничего не записывает + (aborted_lease_lost / commit_rejected_lease_lost); вариантов по-прежнему 3, объектов 4. +""" +import argparse +import subprocess +import time + +from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, container_ids, events, get_att, invariants, ops, + recreate, upload, wait_status, wait_until) + +ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_LEASE_SECONDS": "10", "FAULT_SLOW_SECONDS": "25"} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + args = p.parse_args() + log = Log("lease expiry of a frozen worker and fencing of its late result") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + recreate("worker", ENV) + log.step("workers recreated", env=ENV, workers=[x[:12] for x in container_ids("worker")]) + paused = None + try: + att = upload(c, alice, chat, SAMPLE_PNG, "fault-slow.png") + info = wait_until(lambda: (lambda i: i if i["jobs"][0]["status"] == "processing" else None)( + ops("inspect", att["id"])), 30, 0.5, "processing") + owner = info["jobs"][0]["lease_owner"] + lease1 = info["jobs"][0]["lease_until"] + log.step("claimed", attachment_id=att["id"], lease_owner=owner, lease_until=lease1) + time.sleep(8) + info = ops("inspect", att["id"]) + check(info["jobs"][0]["status"] == "processing" and info["jobs"][0]["lease_owner"] == owner, "long job stolen") + log.step("long-running job keeps its lease (heartbeat)", lease_until=info["jobs"][0]["lease_until"], + status=info["jobs"][0]["status"]) + host = owner.split(":")[0] + paused = [cid for cid in container_ids("worker") if cid.startswith(host)][0] + subprocess.run(["docker", "pause", paused], check=True, capture_output=True) + t_pause = time.monotonic() + log.step("owner container paused (frozen, no heartbeat)", container=paused[:12]) + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90) + took = round(time.monotonic() - t_pause, 1) + info = ops("inspect", att["id"]) + check(ready["status"] == "ready", "not ready after lease expiry") + new_owner = [e["actor"] for e in info["events"] if e["event"] == "claimed"][-1] + check(new_owner.split(":")[0] != host, "must be taken by the other worker") + log.step("lease expired -> requeued -> other worker finished", seconds_after_pause=took, + new_owner=new_owner, events=events(info)) + subprocess.run(["docker", "unpause", paused], check=True, capture_output=True) + paused_id, paused = paused, None + log.step("owner container unpaused") + info = wait_until(lambda: (lambda i: i if {"aborted_lease_lost", "commit_rejected_lease_lost", "error_ignored_lease_lost"} & set(events(i)) + else None)(ops("inspect", att["id"])), 60, 1, "late worker fenced") + inv = invariants(info) + after = get_att(c, bob, att["id"]) + check(inv["variants_ok"] and inv["final_objects_ok"] and after["status"] == "ready", "invariants") + check(after["variants"] == ready["variants"], "late worker changed published result") + log.step("late worker fenced, result unchanged", events=events(info), invariants=inv) + log.save(args.output, attachment_id=att["id"], paused_container=paused_id[:12], events=info["events"], + invariants=inv) + finally: + if paused: + subprocess.run(["docker", "unpause", paused], capture_output=True) + recreate("worker") + print("PASS: frozen worker loses lease, job recovered, late result fenced") + + +if __name__ == "__main__": + main() diff --git a/student-tests/legacy/README.md b/student-tests/legacy/README.md new file mode 100644 index 0000000..2e1866c --- /dev/null +++ b/student-tests/legacy/README.md @@ -0,0 +1,18 @@ +# Проверки lab4–lab5, не перенесённые на транспорт lab7 + +Эти сценарии (нагрузочный failover, медленный клиент, прогрессивный upload, права S3 через +`http://s3:8333`, raw-сокеты к `http://lb:8080` и т. п.) написаны для plaintext-стенда lab5/lab6: +они обращаются к `:8080`/`:8404` без TLS, через `http.client`/сокеты или со старыми общими +credentials. В lab7 этих портов и credentials больше нет (только HTTPS :8443, mTLS внутри, +секреты из OpenBao), поэтому скрипты здесь сохранены как история и **в lab7 не запускаются**. + +Что их заменяет в lab7: + +| Было | Чем покрыто сейчас | +| --- | --- | +| api_negative_lab5, attachments_negative | `tests/smoke.py` (HTTPS) + `student-tests/lab7_api_security.py` (посторонний, лимиты 413, logout/expiry) | +| login_flood | `lab7_api_security.py` — `ratelimit.login_429_with_retry_after` | +| s3_private_check | `lab7_tls_matrix.py` (анонимный доступ 403 по HTTPS) и `lab7_openbao.py` (у worker свой ключ) | +| failover_bench, failover-each.sh, attachment_failover | `make test-failover` по HTTPS для обеих реплик (evidence/test-failover.txt) | +| idempotency_concurrency | `tests/smoke.py`, `tests/load.py` по HTTPS | +| upload_memory, slow_client, progressive_upload, s3_outage, queue_contract_check, variants_check | не перенесены (уровень 4 lab4/lab5); перенос — в «Следующих шагах» REPORT.md | diff --git a/student-tests/legacy/api_negative_lab5.py b/student-tests/legacy/api_negative_lab5.py new file mode 100644 index 0000000..6aefd00 --- /dev/null +++ b/student-tests/legacy/api_negative_lab5.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Регрессии по внешнему аудиту (HTTP-уровень): F02, F05, F08, F09, F17. + + python3 student-tests/api_negative_lab5.py --output evidence/api-negative-lab5.json + + F08 непарный суррогат (JSON "\\ud800") в display_name / password / title / text -> 400, не 500; + то же в filename (%ED%A0%80 — невалидный UTF-8) -> 400 + F17 HEAD / OPTIONS / неизвестный метод -> 405 JSON с X-Request-Id и X-Instance-Id (у HEAD без тела) + F05 PNG с байтом фильтра 5 и 255, interlaced PNG с плохим фильтром -> 415, строка не создана + F09 объект ready-вложения испорчен в S3 (та же длина) -> content 503, а не 200 с чужими байтами + F02 4 progressive 4:4:4 + 1 progressive 4:2:0 + 1 baseline JPEG ~40 Мпикс одновременно -> все 202 + и ready; RestartCount/OOMKilled реплик API не изменились +""" +import argparse +import http.client +import json +import struct +import subprocess +import uuid +import zlib +from urllib.parse import urlsplit + +from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, container_ids, ops, sql, upload, wait_status) + + +def api_state(): + out = {} + for cid in container_ids("api"): + r = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.OOMKilled}}", cid], + capture_output=True, text=True).stdout.split() + out[r[0].lstrip("/")] = {"restarts": int(r[1]), "oom_killed": r[2] == "true"} + return out + + +def raw(base, method, path, body=None, headers=None): + u = urlsplit(base) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=15) + conn.request(method, path, body=body, headers=headers or {}) + r = conn.getresponse() + data = r.read() + conn.close() + return r.status, dict((k.lower(), v) for k, v in r.getheaders()), data + + +def png(w, h, raw_rows, interlace=0): + def chunk(k, d): + return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d) & 0xFFFFFFFF) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, interlace)) + + chunk(b"IDAT", zlib.compress(raw_rows)) + chunk(b"IEND", b"")) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--output") + args = p.parse_args() + log = Log("audit regressions: unicode, protocol errors, PNG filters, download integrity, progressive JPEG") + c = Client(args.base_url, timeout=60) + alice, bob, _eve, chat = c.bootstrap() + res = {} + + # --- F08 ------------------------------------------------------------------------- + bad = '"\\ud800"' + cases = { + "display_name": ("/api/v1/users", None, + '{"username":"u_%s","display_name":%s,"password":"Passw0rd-long-1"}' % (uuid.uuid4().hex[:10], bad)), + "password": ("/api/v1/users", None, + '{"username":"u_%s","display_name":"x","password":%s}' % (uuid.uuid4().hex[:10], '"Passw0rd-long-\\ud800"')), + "chat title": ("/api/v1/chats", alice, '{"title":%s,"member_ids":["%s"]}' % (bad, alice["id"])), + "message text": ("/api/v1/chats/%s/messages" % chat["id"], alice, '{"text":%s}' % bad), + } + f08 = {} + for name, (path, actor, body) in cases.items(): + h = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex} + if actor: + h["Authorization"] = "Bearer " + actor["token"] + st, hd, data = raw(args.base_url, "POST", path, body.encode(), h) + code = json.loads(data)["error"]["code"] if data.startswith(b"{") else None + f08[name] = {"status": st, "code": code} + check(st == 400 and code == "invalid_request", "F08 %s: %s %s" % (name, st, code)) + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=%%ED%%A0%%80.png" % chat["id"], 400, actor=alice, + raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}) + f08["filename %ED%A0%80"] = {"status": r["status"]} + res["F08"] = f08 + log.step("F08 lone surrogates -> 400", **f08) + + # --- F17 ------------------------------------------------------------------------- + f17 = {} + for method, path in (("HEAD", "/health/live"), ("OPTIONS", "/api/v1/chats"), ("BREW", "/health/live"), + ("TRACE", "/health/live")): + st, hd, data = raw(args.base_url, method, path) + item = {"status": st, "x_request_id": bool(hd.get("x-request-id")), "x_instance_id": bool(hd.get("x-instance-id")), + "content_type": hd.get("content-type"), "body_bytes": len(data)} + if method != "HEAD": + item["code"] = json.loads(data)["error"]["code"] + item["request_id_matches"] = json.loads(data)["error"]["request_id"] == hd.get("x-request-id") + check(item["request_id_matches"], "F17 request_id") + check(st == 405 and item["x_request_id"] and item["x_instance_id"], "F17 %s: %s" % (method, item)) + check(method != "HEAD" or len(data) == 0, "HEAD must have no body") + f17[method] = item + res["F17"] = f17 + log.step("F17 unsupported methods -> 405 JSON", **f17) + + # --- F05 ------------------------------------------------------------------------- + f05 = {} + import sys + sys.path.insert(0, str(__import__("lab5lib").ROOT / "src")) + import images + w, h = 13, 11 + passes = images._png_passes(w, h, 24, 1) + adam = bytearray(b"".join((b"\x00" + bytes(rb - 1)) * rows for rows, rb in passes)) + adam[sum(rr * bb for rr, bb in passes[:4]) + passes[4][1] * 2] = 9 + samples = {"filter 5 (1x1)": png(1, 1, b"\x05\x01\x02\x03"), + "filter 255 in row 57 (10x100)": png(10, 100, b"".join((b"\xff" if i == 57 else b"\x00") + bytes(30) + for i in range(100))), + "Adam7, filter 9 in pass 5": png(w, h, bytes(adam), interlace=1)} + for name, data in samples.items(): + fname = "badfilter-%s.png" % uuid.uuid4().hex[:8] + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], fname), 415, actor=alice, + raw=data, headers={"Content-Type": "image/png"}) + rows = int(sql("SELECT count(*) FROM attachments WHERE filename = '%s'" % fname)[0][0]) + check(rows == 0, "F05 row created") + f05[name] = {"status": r["status"], "bytes": len(data), "rows_created": rows} + res["F05"] = f05 + log.step("F05 bad PNG filters -> 415, no job", **f05) + + # --- F09 ------------------------------------------------------------------------- + att = upload(c, alice, chat, SAMPLE_PNG, "integrity.png") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 60) + check(ready["status"] == "ready", "F09 not ready") + key = ops("inspect", att["id"])["attachment"]["object_key"] + compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key) + st, hd, data = raw(args.base_url, "GET", "/api/v1/attachments/%s/content" % att["id"], + headers={"Authorization": "Bearer " + bob["token"]}) + thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob) + tv = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + import hashlib + res["F09"] = {"attachment_id": att["id"], "tampered_original_status": st, + "code": json.loads(data)["error"]["code"] if st >= 400 else None, + "untouched_thumbnail_status": thumb["status"], + "thumbnail_sha_ok": hashlib.sha256(thumb["bytes"]).hexdigest() == tv["sha256"]} + check(st == 503 and res["F09"]["thumbnail_sha_ok"], "F09: %s" % res["F09"]) + log.step("F09 corrupted object -> 503, intact variant still 200", **res["F09"]) + + # --- F02 ------------------------------------------------------------------------- + before = api_state() + proc = compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/progressive_upload.py", + "--base-url", "http://lb:8080", "--output", "/app/evidence/progressive-upload.json", ok=False, + timeout=600) + after = api_state() + res["F02"] = {"checks_exit": proc.returncode, "api_before": before, "api_after": after, + "tail": proc.stdout.strip().splitlines()[-1:]} + check(proc.returncode == 0 and before == after and not any(v["oom_killed"] for v in after.values()), + "F02: %s" % res["F02"]) + log.step("F02 concurrent ~40 Mpx progressive JPEG: all ready, API not restarted", **res["F02"]) + log.save(args.output, results=res) + print("PASS: F02, F05, F08, F09, F17 regressions") + + +if __name__ == "__main__": + main() diff --git a/student-tests/legacy/attachment_failover.py b/student-tests/legacy/attachment_failover.py new file mode 100644 index 0000000..f48a4c1 --- /dev/null +++ b/student-tests/legacy/attachment_failover.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Upload -> остановка ИМЕННО обслужившей upload реплики -> download через другую. + + python3 student-tests/attachment_failover.py [--output evidence/attachment-failover.json] + +Для каждой API-реплики по очереди: загрузить файл так, чтобы его обслужила эта реплика +(round-robin LB, повторяем до попадания), остановить её scripts/stop-one.sh +(SIGTERM, контейнер остаётся остановленным), затем 5 раз скачать оригинал через LB и +проверить SHA-256 и что отвечает уже другая реплика. Восстановление — scripts/start-all.sh. +Доказывает, что на API нет обязательного локального файла оригинала. +""" +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check # noqa: E402 + +CONTENT = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--output") + args = ap.parse_args() + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + instances = set() + for _ in range(10): + instances.add(c.call("GET", "/health/ready")["headers"]["X-Instance-Id"]) + check(len(instances) >= 2, "need >=2 replicas") + rounds = [] + for target in sorted(instances): + for _ in range(20): # round-robin: попадаем в нужную реплику за пару попыток + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=failover.png" % chat["id"], 202, actor=alice, + raw=CONTENT, headers={"Content-Type": "image/png"}, schema="Attachment") + if r["headers"]["X-Instance-Id"] == target: + break + else: + raise SystemExit("could not hit replica " + target) + att = c.wait_ready(alice, r["json"]) # lab5: 202 queued -> ждём ready, затем отказ реплики + stop = subprocess.run([str(ROOT / "scripts/stop-one.sh")], env=dict(os.environ, TARGET_INSTANCE_ID=target), + capture_output=True, text=True, timeout=60) + check(stop.returncode == 0, "stop-one failed") + downloads = [] + for _ in range(5): + d = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob) + downloads.append({"instance": d["headers"]["X-Instance-Id"], + "sha256_ok": hashlib.sha256(d["bytes"]).hexdigest() == att["sha256"] == hashlib.sha256(CONTENT).hexdigest(), + "ms": round(d["elapsed"] * 1000, 1)}) + meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + ok = all(x["sha256_ok"] and x["instance"] != target for x in downloads) and meta["status"] == "ready" + rounds.append({"uploaded_by": target, "stopped": target, "downloads": downloads, "pass": ok}) + print(json.dumps(rounds[-1]), flush=True) + subprocess.run([str(ROOT / "scripts/start-all.sh")], capture_output=True, timeout=120, check=True) + time.sleep(4) # HAProxy: rise 2 * inter 1s + summary = {"replicas": sorted(instances), "rounds": rounds, "pass": all(r["pass"] for r in rounds)} + print(json.dumps({"pass": summary["pass"], "rounds": len(rounds)})) + if args.output: + Path(args.output).write_text(json.dumps(summary, indent=1)) + return 0 if summary["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/attachments_negative.py b/student-tests/legacy/attachments_negative.py new file mode 100644 index 0000000..87df2a0 --- /dev/null +++ b/student-tests/legacy/attachments_negative.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Негативные и граничные сценарии вложений (из lab4; в lab5 upload -> 202, затем ready). Запуск на учебном стенде: + + python3 student-tests/attachments_negative.py [--base-url http://localhost:8080] [--output evidence/attachments-negative.json] + +Использует публичный tests/client.py: каждый ответ проверяется на код, X-Request-Id, +X-Instance-Id и (для ошибок) формат {"error":{code,message,request_id}}. Отдельно +проверяется, что ни один пользовательский ввод не дал 5xx. Все файлы синтетические и +генерируются на лету (большие в Git не хранятся). +""" +import argparse +import hashlib +import json +import os +import struct +import sys +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 + +PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() +JPEG = (ROOT / "student-tests/fixtures/sample.jpg").read_bytes() +MAX_BYTES = 10 * 1024 * 1024 +results = [] + + +def chunk(ctype, data): + return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data) & 0xFFFFFFFF) + + +def png(width, height, raw, color_type=2, depth=8): + ihdr = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", raw) + chunk(b"IEND", b"") + + +def noise_png(target_bytes): + """Несжимаемый RGB PNG размером чуть меньше target_bytes (zlib level 0).""" + width = 1024 + row = 1 + width * 3 + height = (target_bytes - 200) // (row + 6 * 0 + 1) + while True: + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + data = png(width, height, zlib.compress(raw, 0)) + if len(data) <= target_bytes: + return data + height -= 16 + + +class Suite: + def __init__(self, c): + self.c = c + + def case(self, name, method, path, expected, actor=None, **kw): + try: + r = self.c.call(method, path, expected, actor=actor, **kw) + code = r["json"]["error"]["code"] if r["json"] and "error" in r["json"] else None + results.append({"case": name, "expected": expected, "status": r["status"], "error_code": code, "pass": True}) + return r + except Failure as exc: + results.append({"case": name, "expected": expected, "error": str(exc), "pass": False}) + return None + + def upload(self, name, actor, chat, data, expected, ctype="image/png", filename="f.png", raw_query=None): + q = raw_query if raw_query is not None else "filename=" + quote(filename, safe="") + return self.case(name, "POST", "/api/v1/chats/%s/attachments?%s" % (chat["id"], q), expected, + actor=actor, raw=data, headers={"Content-Type": ctype}, + schema="Attachment" if expected == 202 else None) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--output") + args = ap.parse_args() + c = Client(args.base_url, timeout=60) + s = Suite(c) + alice, bob, eve, chat = c.bootstrap() + chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "второй чат", "member_ids": [alice["id"], bob["id"]]}, schema="Chat")["json"] + eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve, + body={"title": "чат Евы", "member_ids": [eve["id"]]}, schema="Chat")["json"] + + # --- позитивные контрольные --- + ok_png = s.upload("valid PNG fixture", alice, chat, PNG, 202) + ok_jpg = s.upload("valid JPEG", bob, chat, JPEG, 202, ctype="image/jpeg", filename="photo.jpg") + uni = s.upload("unicode filename", alice, chat, PNG, 202, filename="фото отпуска 🙂.png") + if uni: + check(uni["json"]["filename"] == "фото отпуска 🙂.png", "unicode filename changed") + def ready(r): # lab5: upload -> 202 queued, байты доступны после ready + if r: + c.wait_ready(alice if r["json"]["owner_id"] == alice["id"] else bob, r["json"]) + for r in (ok_png, ok_jpg): + ready(r) + if ok_jpg: + d = c.call("GET", "/api/v1/attachments/%s/content" % ok_jpg["json"]["id"], actor=alice) + check(d["bytes"] == JPEG and d["headers"].get_content_type() == "image/jpeg", "jpeg download mismatch") + check(d["headers"].get("X-Content-Type-Options") == "nosniff", "nosniff missing") + near = noise_png(MAX_BYTES) + big = s.upload("PNG near 10 MiB limit (%d B)" % len(near), alice, chat, near, 202, filename="big.png") + if big: + check(big["json"]["sha256"] == hashlib.sha256(near).hexdigest(), "big sha mismatch") + ready(big) + d = c.call("GET", "/api/v1/attachments/%s/content" % big["json"]["id"], actor=bob) + check(hashlib.sha256(d["bytes"]).hexdigest() == big["json"]["sha256"], "big download mismatch") + + # --- размер и пиксели --- + s.upload("too large: 10 MiB + 1 B", alice, chat, near + b"\0" * (MAX_BYTES + 1 - len(near)), 413) + s.upload("pixel bomb header 100000x100000", alice, chat, png(100000, 100000, zlib.compress(b"\0" * 1000)), 413) + s.upload("PNG 7000x6000 = 42 Mpx (> 40 Mpx)", alice, chat, png(7000, 6000, zlib.compress(b"\0" * 1000)), 413) + s.upload("zlib bomb: 100x100 header, 50 MB inflated", alice, chat, + png(100, 100, zlib.compress(b"\0" * 50_000_000, 9), color_type=0), 415) + + # --- повреждённые изображения --- + flipped = bytearray(PNG); flipped[5000] ^= 0xFF + s.upload("corrupted PNG (byte flipped, bad CRC)", alice, chat, bytes(flipped), 415) + s.upload("truncated PNG", alice, chat, PNG[:-4000], 415) + s.upload("PNG without IEND", alice, chat, PNG[:-12], 415) + s.upload("PNG with appended ZIP (polyglot)", alice, chat, PNG + b"PK\x03\x04" + os.urandom(64), 415) + s.upload("truncated JPEG", alice, chat, JPEG[:len(JPEG) // 2], 415, ctype="image/jpeg") + s.upload("JPEG header only", alice, chat, JPEG[:200], 415, ctype="image/jpeg") + s.upload("empty body", alice, chat, b"", 415) + + # --- подмена MIME --- + s.upload("JPEG bytes declared image/png", alice, chat, JPEG, 415) + s.upload("PNG bytes declared image/jpeg", alice, chat, PNG, 415, ctype="image/jpeg") + s.upload("text declared image/png", alice, chat, b"" * 4, 415) + s.upload("GIF declared image/png", alice, chat, b"GIF89a" + os.urandom(200), 415) + s.upload("PNG declared text/html", alice, chat, PNG, 415, ctype="text/html") + s.upload("PNG declared image/svg+xml", alice, chat, PNG, 415, ctype="image/svg+xml") + s.upload("multipart/form-data instead of raw", alice, chat, b"--x\r\n\r\n" + PNG + b"\r\n--x--", 415, + ctype="multipart/form-data; boundary=x") + + # --- опасные filename --- + for label, query in (("../etc/passwd", "filename=" + quote("../passwd.png", safe="")), + ("slash", "filename=" + quote("a/b.png", safe="")), + ("backslash", "filename=" + quote("a\\b.png", safe="")), + ("NUL", "filename=a%00.png"), + ("newline", "filename=a%0A.png"), + ("bidi override U+202E", "filename=" + quote("‮gnp.exe", safe="")), + ("..", "filename=.."), + ("empty", "filename="), + ("whitespace only", "filename=%20%20"), + ("256 code points", "filename=" + quote("я" * 256, safe="")), + ("missing filename", ""), + ("filename twice", "filename=a.png&filename=b.png"), + ("invalid UTF-8", "filename=%FF%FE.png")): + s.upload("filename: " + label, alice, chat, PNG, 400, raw_query=query) + + # --- доступ и чужие объекты --- + eve_att = s.upload("eve uploads to her own chat", eve, eve_chat, PNG, 202) + s.upload("non-member uploads to alice chat", eve, chat, PNG, 404) + s.case("upload without session", "POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 401, + raw=PNG, headers={"Content-Type": "image/png"}) + s.upload("upload to unknown chat", alice, {"id": str(uuid.uuid4())}, PNG, 404) + if eve_att and ok_png: + eid, aid = eve_att["json"]["id"], ok_png["json"]["id"] + s.case("foreign attachment metadata", "GET", "/api/v1/attachments/" + eid, 404, actor=alice) + s.case("foreign attachment content", "GET", "/api/v1/attachments/%s/content" % eid, 404, actor=alice) + s.case("eve reads alice attachment", "GET", "/api/v1/attachments/%s/content" % aid, 404, actor=eve) + key = lambda: {"Idempotency-Key": uuid.uuid4().hex} # noqa: E731 + s.case("message with foreign attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404, + actor=alice, body={"text": "x", "attachment_ids": [eid]}, headers=key()) + s.case("message mixing own and foreign ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404, + actor=alice, body={"text": "x", "attachment_ids": [aid, eid]}, headers=key()) + s.case("attachment of OTHER chat of the same user", "POST", "/api/v1/chats/%s/messages" % chat2["id"], 404, + actor=alice, body={"text": "x", "attachment_ids": [aid]}, headers=key()) + s.case("unknown attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404, actor=alice, + body={"text": "x", "attachment_ids": [str(uuid.uuid4())]}, headers=key()) + s.case("malformed attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, + body={"text": "x", "attachment_ids": ["not-a-uuid"]}, headers=key()) + s.case("duplicate attachment_ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, + body={"text": "x", "attachment_ids": [aid, aid]}, headers=key()) + s.case("11 attachment_ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, + body={"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, headers=key()) + s.case("attachment_ids not a list", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, + body={"text": "x", "attachment_ids": aid}, headers=key()) + s.case("empty text without attachments", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, + actor=alice, body={"text": " "}, headers=key()) + m = s.case("empty text with attachment (bob links alice's file)", "POST", + "/api/v1/chats/%s/messages" % chat["id"], 201, actor=bob, + body={"text": "", "attachment_ids": [aid]}, headers=key(), schema="Message") + if m: + check(m["json"]["attachment_ids"] == [aid], "attachment_ids lost") + k = key() + s.case("idempotent send with attachments", "POST", "/api/v1/chats/%s/messages" % chat["id"], 201, + actor=alice, body={"attachment_ids": [aid], "text": "повтор"}, headers=k, schema="Message") + s.case("same key, same body (key order changed)", "POST", "/api/v1/chats/%s/messages" % chat["id"], 201, + actor=alice, body={"text": "повтор", "attachment_ids": [aid]}, headers=k, schema="Message") + s.case("same key, different attachment_ids -> 409", "POST", "/api/v1/chats/%s/messages" % chat["id"], 409, + actor=alice, body={"text": "повтор", "attachment_ids": []}, headers=k) + s.case("variant=thumbnail (ready, lab5)", "GET", + "/api/v1/attachments/%s/content?variant=thumbnail" % aid, 200, actor=bob) + s.case("variant=bogus", "GET", "/api/v1/attachments/%s/content?variant=bogus" % aid, 400, actor=bob) + s.case("variant=original", "GET", "/api/v1/attachments/%s/content?variant=original" % aid, 200, actor=bob) + s.case("malformed attachment id in path", "GET", "/api/v1/attachments/zzz", 400, actor=bob) + s.case("unknown attachment id", "GET", "/api/v1/attachments/" + str(uuid.uuid4()), 404, actor=bob) + + failed = [r for r in results if not r["pass"]] + server_errors = [r for r in results if r.get("status", 0) >= 500] + for r in results: + print(("PASS " if r["pass"] else "FAIL ") + "%-55s -> %s %s" % ( + r["case"], r.get("status", "-"), r.get("error_code") or r.get("error", ""))) + summary = {"cases": len(results), "failed": len(failed), "server_errors_5xx": len(server_errors), + "pass": not failed and not server_errors} + print(json.dumps(summary, ensure_ascii=False)) + if args.output: + Path(args.output).write_text(json.dumps({"summary": summary, "results": results}, ensure_ascii=False, indent=1)) + return 0 if summary["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/failover-each.sh b/student-tests/legacy/failover-each.sh new file mode 100755 index 0000000..9b08778 --- /dev/null +++ b/student-tests/legacy/failover-each.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Уровень 4: ПОСЛЕДОВАТЕЛЬНО останавливает КАЖДУЮ API-реплику под нагрузкой, +# восстанавливая предыдущую перед следующей. Режимы: +# stop — docker stop (SIGTERM -> graceful drain), штатная остановка/деплой; +# kill — docker kill (SIGKILL), внезапное падение процесса без drain. +# Результаты: evidence/failover--.json (+ .action.log). +# Использование: scripts/failover-each.sh [stop|kill] ... (по умолчанию: stop kill) +set -eu +cd "$(dirname "$0")/.." +MODES=${*:-stop kill} +CONCURRENCY=${CONCURRENCY:-8} +DURATION=${DURATION:-40} +mkdir -p evidence + +wait_all_up() { + # API-контейнеры healthy и HAProxy видит их все UP (rise 2 * inter 1s). + ./scripts/start-all.sh >/dev/null + n=$(docker compose ps -q api | wc -l | tr -d ' ') + for _ in $(seq 1 60); do + up=$(curl -s 'http://127.0.0.1:8404/stats;csv' | awk -F, '$1=="be_api" && $18=="UP"' | wc -l | tr -d ' ') + [ "$up" -ge "$n" ] && return 0 + sleep 1 + done + echo "HAProxy did not mark all $n api servers UP" >&2; return 1 +} + +wait_all_up +# Список фиксируется один раз: остановленная реплика пропала бы из `docker compose ps`. +NAMES=$(docker compose ps --format '{{.Name}}' api | sort) +for mode in $MODES; do + for name in $NAMES; do + wait_all_up + cid=$(docker inspect -f '{{.Id}}' "$name"); short=$(echo "$cid" | cut -c1-12) + case $mode in + stop) action="docker stop -t 20 $cid" ;; + kill) action="docker kill $cid" ;; + *) echo "unknown mode $mode" >&2; exit 2 ;; + esac + echo "== $mode $name ($short) $(date -u +%FT%TZ)" + out=evidence/failover-$mode-$name.json + python3 student-tests/failover_bench.py --label "$mode $name" \ + --concurrency "$CONCURRENCY" --duration "$DURATION" --action-at 10 \ + --action "$action" --target-instance "$short" \ + --action-log "${out%.json}.action.log" --output "$out" \ + | python3 -c 'import json,sys; t=sys.stdin.read(); s=json.loads(t[:t.rindex("}")+1]); print(json.dumps({k:s[k] for k in ("baseline_ok_rps","post_action_ok_rps","capacity_ratio","errors_total","errors_by_kind","error_window_s","recovery_s","target_last_seen_s","lb_events","lost_confirmed","duplicate_texts","pass")},ensure_ascii=False)); sys.exit(0 if s["pass"] else 1)' + echo "-- state after action:"; docker compose ps -a --format '{{.Name}} {{.Status}}' api + done +done +wait_all_up +echo "== restored: $(docker compose ps --format '{{.Name}} {{.Status}}' api | tr '\n' ';')" diff --git a/student-tests/legacy/failover_bench.py b/student-tests/legacy/failover_bench.py new file mode 100644 index 0000000..87ffd2a --- /dev/null +++ b/student-tests/legacy/failover_bench.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Отказ/замена API-реплики ПОД НАГРУЗКОЙ: RPS, ошибки, время обнаружения и восстановления (lab3, уровень 4). + +Что делает: + * CONCURRENCY потоков в closed-loop шлют смесь запросов через LB: POST сообщения + (с Idempotency-Key) и GET истории (limit=20) в пропорции 1:1; + * после неуспешного/неопределённого POST поток повторяет ТОТ ЖЕ ключ и тело (как + требует контракт), поэтому подтверждённых дублей быть не должно; + * на секунде --action-at запускается --action (например, docker stop/kill контейнера + или scripts/rolling-update.sh), его stdout/stderr пишутся в отдельный лог; + * параллельно каждые 200 мс опрашивается CSV статистики HAProxy: фиксируются моменты + UP -> DOWN / DOWN -> UP серверов backend (время обнаружения отказа LB); + * в конце читается вся история чата (тем же токеном) и проверяется: все подтверждённые + сообщения на месте, дублей нет. + +Метрики (все времена — от старта action): + errors_total / error_rate — неуспешные попытки (HTTP != ожидаемого, timeout, разрыв); + error_window_s — от первой до последней ошибки после старта action; + recovery_s — конец окна ошибок (дальше все запросы успешны), 0 если ошибок нет; + post_action_ok_rps/capacity_ratio — успешный RPS после действия и его доля от базового; + lb_down_detected_s — когда HAProxy пометил сервер DOWN; + target_last_seen_s — последний ответ от остановленной реплики. + +Ограничения: closed-loop генератор (coordinated omission), клиент на том же хосте, что и +стенд; цифры RPS сравнимы только между прогонами этого же скрипта на этом же ноутбуке. +""" +import argparse +import csv +import io +import json +import shlex +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from collections import Counter, defaultdict +from pathlib import Path +from urllib.parse import quote + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client # noqa: E402 + +TIMEOUT = 2.0 + + +def percentile(values, p): + if not values: + return None + s = sorted(values) + return round(s[min(len(s) - 1, int(round(p / 100 * (len(s) - 1))))] * 1000, 1) + + +class Bench: + def __init__(self, a): + self.a = a + self.base = a.base_url.rstrip("/") + self.lock = threading.Lock() + self.t0 = None + self.stop = threading.Event() + self.per_sec = defaultdict(lambda: {"ok": 0, "err": 0, "inst": Counter(), "ver": Counter()}) + self.errors = [] # (t, kind) + self.lat = [] # успешные задержки, с + self.confirmed = {} # message id -> text + self.retried_posts = 0 + self.lb_events = [] + self.inst_last_seen = {} + + def now(self): + return time.monotonic() - self.t0 + + def record(self, ok, dt, inst=None, ver=None, kind=None): + t = self.now() + with self.lock: + b = self.per_sec[int(t)] + if ok: + b["ok"] += 1 + self.lat.append(dt) + if inst: + b["inst"][inst] += 1 + self.inst_last_seen[inst] = t + if ver: + b["ver"][ver] += 1 + else: + b["err"] += 1 + self.errors.append((round(t, 3), kind)) + + def request(self, method, path, body=None, headers=None, expected=200): + h = {"Authorization": "Bearer " + self.token} + h.update(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + req = urllib.request.Request(self.base + path, data=data, method=method, headers=h) + start = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + payload = json.loads(r.read()) + ok = r.status == expected + self.record(ok, time.monotonic() - start, r.headers.get("X-Instance-Id"), + r.headers.get("X-App-Version"), None if ok else "http_%d" % r.status) + return payload if ok else None + except urllib.error.HTTPError as e: + self.record(False, 0, kind="http_%d" % e.code) + except Exception as e: # noqa: BLE001 + self.record(False, 0, kind=type(e).__name__) + return None + + def worker(self, n): + chat_path = "/api/v1/chats/" + self.chat["id"] + "/messages" + key = text = None + i = 0 + while not self.stop.is_set(): + i += 1 + if i % 2: + if key is None: # новый логический запрос; после ошибки ключ НЕ меняется + key = "bench-%d-%s" % (n, uuid.uuid4().hex) + text = "Нагрузка " + key + else: + with self.lock: + self.retried_posts += 1 + msg = self.request("POST", chat_path, {"text": text}, {"Idempotency-Key": key}, 201) + if msg is not None: + with self.lock: + self.confirmed[msg["id"]] = msg["text"] + key = None + else: + self.request("GET", chat_path + "?limit=20") + + def poll_lb(self): + prev = {} + while not self.stop.is_set(): + try: + with urllib.request.urlopen(self.a.stats_url, timeout=1) as r: + rows = list(csv.reader(io.StringIO(r.read().decode().lstrip("# ")))) + head = rows[0] + ix = {k: head.index(k) for k in ("pxname", "svname", "status", "addr")} + for row in rows[1:]: + if len(row) <= ix["addr"] or row[ix["pxname"]] != "be_api" or row[ix["svname"]] == "BACKEND": + continue + sv, st, addr = row[ix["svname"]], row[ix["status"]].split(" ")[0], row[ix["addr"]] + if sv in prev and prev[sv] != (st, addr): + self.lb_events.append({"t": round(self.now(), 2), "server": sv, "addr": addr, + "from": prev[sv][0], "to": st}) + prev[sv] = (st, addr) + except Exception: # noqa: BLE001 — статистика вспомогательная + pass + time.sleep(0.2) + + def history(self): + c, path, items = self.client, "/api/v1/chats/" + self.chat["id"] + "/messages?limit=100", [] + while True: + page = c.call("GET", path, actor=self.alice, schema="MessagePage")["json"] + items += page["items"] + if page["next_cursor"] is None: + return items + path = "/api/v1/chats/" + self.chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="") + + def run(self): + a = self.a + self.client = Client(self.base, timeout=10) + self.alice, _bob, _eve, self.chat = self.client.bootstrap() + self.token = self.alice["token"] + self.t0 = time.monotonic() + threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(a.concurrency)] + threads.append(threading.Thread(target=self.poll_lb, daemon=True)) + for t in threads: + t.start() + time.sleep(a.action_at) + action = {"command": a.action, "started_s": round(self.now(), 3)} + log = open(a.action_log, "w", encoding="utf-8") if a.action_log else subprocess.DEVNULL + proc = subprocess.Popen(shlex.split(a.action), stdout=log, stderr=subprocess.STDOUT) + rc = proc.wait(timeout=a.duration) + action.update(finished_s=round(self.now(), 3), exit_code=rc) + remaining = a.duration - self.now() + if remaining > 0: + time.sleep(remaining) + self.stop.set() + for t in threads: + t.join(timeout=TIMEOUT + 1) + return self.summarize(action) + + def summarize(self, action): + a, t_act = self.a, action["started_s"] + secs = sorted(self.per_sec) + base = [self.per_sec[s]["ok"] for s in secs if 1 <= s < int(t_act)] # 1-я секунда — прогрев + baseline = sum(base) / len(base) if base else 0 + after = [(round(t - t_act, 3), k) for t, k in self.errors if t >= t_act] + # Восстановление = конец окна ошибок: после этого момента все запросы до конца + # прогона успешны. Если ошибок не было — 0. Мощность после отказа меряем отдельно: + # при меньшем числе реплик RPS законно ниже (лимиты CPU на реплику). + recovery = after[-1][0] if after else 0.0 + tail_secs = [s for s in secs if int(action.get("finished_s", t_act)) + 2 <= s < a.duration - 1] + post = [self.per_sec[s]["ok"] for s in tail_secs] + post_rps = sum(post) / len(post) if post else 0 + writes_after = any(self.per_sec[s]["ok"] > 0 for s in tail_secs) + target = a.target_instance + hist = self.history() + ids = [m["id"] for m in hist] + texts = [m["text"] for m in hist] + lost = [i for i in self.confirmed if i not in set(ids)] + timeline = [{"t": s, "ok": self.per_sec[s]["ok"], "err": self.per_sec[s]["err"], + "instances": dict(self.per_sec[s]["inst"]), "versions": dict(self.per_sec[s]["ver"])} + for s in secs] + total_ok = sum(b["ok"] for b in self.per_sec.values()) + total_err = len(self.errors) + summary = { + "label": a.label, + "concurrency": a.concurrency, "duration_s": a.duration, "request_timeout_s": TIMEOUT, + "action": action, + "baseline_ok_rps": round(baseline, 1), + "overall_ok_rps": round(total_ok / a.duration, 1), + "requests_ok": total_ok, "errors_total": total_err, + "error_rate": round(total_err / max(1, total_ok + total_err), 5), + "errors_by_kind": dict(Counter(k for _, k in self.errors)), + "error_window_s": [after[0][0], after[-1][0]] if after else None, + "recovery_s": recovery, + "post_action_ok_rps": round(post_rps, 1), + "capacity_ratio": round(post_rps / baseline, 2) if baseline else None, + "latency_ok_ms": {"p50": percentile(self.lat, 50), "p95": percentile(self.lat, 95), + "p99": percentile(self.lat, 99)}, + "target_instance": target, + "target_last_seen_s": (round(self.inst_last_seen[target] - t_act, 3) + if target in self.inst_last_seen else None), + "lb_events": [dict(e, t=round(e["t"] - t_act, 2)) for e in self.lb_events], + "posts_retried_same_key": self.retried_posts, + "confirmed_messages": len(self.confirmed), + "history_messages": len(hist), "lost_confirmed": len(lost), + "duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)), + "timeline": timeline, + } + summary["pass"] = (summary["lost_confirmed"] == 0 and summary["duplicate_ids"] == 0 + and summary["duplicate_texts"] == 0 and action["exit_code"] == 0 + and writes_after and recovery <= 10) + return summary + + +def main(): + p = argparse.ArgumentParser(description="Отказ/замена реплики под нагрузкой") + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--stats-url", default="http://127.0.0.1:8404/stats;csv") + p.add_argument("--concurrency", type=int, default=8) + p.add_argument("--duration", type=float, default=40) + p.add_argument("--action-at", type=float, default=10) + p.add_argument("--action", required=True, help="команда без shell, например 'docker stop -t 20 '") + p.add_argument("--action-log") + p.add_argument("--target-instance", help="X-Instance-Id реплики, которую останавливает action") + p.add_argument("--label", default="") + p.add_argument("--output") + a = p.parse_args() + s = Bench(a).run() + brief = {k: v for k, v in s.items() if k != "timeline"} + print(json.dumps(brief, ensure_ascii=False, indent=2)) + if a.output: + Path(a.output).write_text(json.dumps(s, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS" if s["pass"] else "FAIL") + return 0 if s["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/idempotency_concurrency.py b/student-tests/legacy/idempotency_concurrency.py new file mode 100644 index 0000000..882d233 --- /dev/null +++ b/student-tests/legacy/idempotency_concurrency.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Конкурентная идемпотентность через разные реплики (lab3, уровни 3–4). + +Сценарий на каждый раунд: + A) PARALLEL запросов одновременно отправляют ОДИН И ТОТ ЖЕ ключ и одинаковое тело. + Ожидаем: все 201, у всех один и тот же Message.id, ответы пришли от >=2 реплик, + в истории ровно одно сообщение с этим текстом. + B) Тот же ключ, но половина запросов с другим телом. Ожидаем: побеждает ровно одно + тело; запросы с ним — 201 и один id, запросы с другим телом — 409; дублей нет. + +Каждый запрос — отдельное TCP-соединение, поэтому HAProxy (roundrobin) раскладывает их +по репликам. Токены только в памяти; синтетические пользователи создаются заново. + + python3 student-tests/idempotency_concurrency.py --rounds 20 --parallel 16 +""" +import argparse +import json +import sys +import threading +import urllib.error +import urllib.request +import uuid +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client # noqa: E402 (публичный клиент тестов: bootstrap пользователей/чата) + + +def post(base, token, chat_id, key, text, barrier, out, idx): + data = json.dumps({"text": text}).encode() + req = urllib.request.Request( + base + "/api/v1/chats/" + chat_id + "/messages", data=data, method="POST", + headers={"Authorization": "Bearer " + token, "Content-Type": "application/json", + "Idempotency-Key": key, "Connection": "close"}) + barrier.wait() + try: + with urllib.request.urlopen(req, timeout=10) as r: + out[idx] = (r.status, json.loads(r.read()), r.headers.get("X-Instance-Id"), text) + except urllib.error.HTTPError as e: + out[idx] = (e.code, None, e.headers.get("X-Instance-Id"), text) + except Exception as e: # noqa: BLE001 — считаем любую сетевую ошибку + out[idx] = (0, repr(e), None, text) + + +def burst(base, token, chat_id, key, texts): + barrier = threading.Barrier(len(texts)) + out = [None] * len(texts) + threads = [threading.Thread(target=post, args=(base, token, chat_id, key, t, barrier, out, i)) + for i, t in enumerate(texts)] + for t in threads: + t.start() + for t in threads: + t.join() + return out + + +def history_texts(c, actor, chat): + from urllib.parse import quote + texts, path = [], "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + while True: + page = c.call("GET", path, actor=actor, schema="MessagePage")["json"] + texts += [m["text"] for m in page["items"]] + if page["next_cursor"] is None: + return texts + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="") + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--rounds", type=int, default=20) + p.add_argument("--parallel", type=int, default=16) + p.add_argument("--output") + a = p.parse_args() + base = a.base_url.rstrip("/") + c = Client(base) + alice, _bob, _eve, chat = c.bootstrap() + token = alice["token"] + problems, instances = [], Counter() + stats = {"same_body": Counter(), "conflicting_body": Counter()} + + for rnd in range(a.rounds): + # A) одинаковое тело + key, text = "same-" + uuid.uuid4().hex, "Конкурентный повтор " + uuid.uuid4().hex + res = burst(base, token, chat["id"], key, [text] * a.parallel) + stats["same_body"].update(str(r[0]) for r in res) + ids = {r[1]["id"] for r in res if r[0] == 201} + seen = {r[2] for r in res if r[2]} + instances.update(r[2] for r in res if r[2]) + if any(r[0] != 201 for r in res) or len(ids) != 1: + problems.append("round %d same-body: statuses=%s ids=%d" % (rnd, Counter(r[0] for r in res), len(ids))) + if len(seen) < 2: + problems.append("round %d: only %d replica(s) answered" % (rnd, len(seen))) + + # B) тот же ключ, два разных тела вперемешку + key = "mixed-" + uuid.uuid4().hex + t1, t2 = "Тело A " + key, "Тело B " + key + res = burst(base, token, chat["id"], key, [t1, t2] * (a.parallel // 2)) + stats["conflicting_body"].update(str(r[0]) for r in res) + winners = {r[3] for r in res if r[0] == 201} + ids = {r[1]["id"] for r in res if r[0] == 201} + bad = [r[0] for r in res if r[0] not in (201, 409)] + losers_ok = all(r[0] == 409 for r in res if r[3] not in winners) + if len(winners) != 1 or len(ids) != 1 or bad or not losers_ok: + problems.append("round %d mixed-body: winners=%d ids=%d bad=%s" % (rnd, len(winners), len(ids), bad)) + + texts = history_texts(c, alice, chat) + dupes = [t for t, n in Counter(texts).items() if n > 1] + if dupes: + problems.append("duplicated messages in history: %d" % len(dupes)) + summary = { + "rounds": a.rounds, "parallel_per_burst": a.parallel, + "requests_total": a.rounds * a.parallel * 2, + "status_counts": {k: dict(v) for k, v in stats.items()}, + "replicas_answered": len(instances), "requests_per_replica": sorted(instances.values()), + "messages_in_history": len(texts), "expected_messages": a.rounds * 2, + "duplicates": len(dupes), "problems": problems, + } + if len(texts) != a.rounds * 2: + problems.append("history has %d messages, expected %d" % (len(texts), a.rounds * 2)) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + if a.output: + Path(a.output).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS" if not problems else "FAIL") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/login_flood.py b/student-tests/legacy/login_flood.py new file mode 100644 index 0000000..eae87be --- /dev/null +++ b/student-tests/legacy/login_flood.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Шторм входов: много параллельных POST /auth/sessions с неверным паролем через LB. + +Зачем: scrypt (N=2^15, r=8) берёт ~32 MiB на одну проверку пароля, лимит API-реплики +256 MiB. Без ограничения параллельных проверок несколько одновременных входов +приводят к OOM-kill реплики. Скрипт показывает, переживают ли реплики такой шторм. + +Что измеряется: + * статусы ответов шторма (ожидаем 401 и, при перегрузке, 503 с Retry-After) и + ответы не в JSON-формате ошибки контракта (например HTML-страницы HAProxy); + * «жертва»: параллельный поток раз в 50 мс читает /users/me действующим токеном — + обычные пользователи не должны получать ошибки из-за шторма входов; + * состояние API-контейнеров до/после: OOMKilled, RestartCount, StartedAt. + +Пример: python3 student-tests/login_flood.py --parallel 24 --duration 20 --output evidence/login-flood.json +""" +import argparse +import base64 +import json +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +from collections import Counter + +TIMEOUT = 10.0 + + +def http(base, method, path, headers=None, body=None): + data = json.dumps(body).encode() if body is not None else None + h = dict(headers or {}) + if data is not None: + h["Content-Type"] = "application/json" + req = urllib.request.Request(base + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + return r.status, r.headers, r.read() + except urllib.error.HTTPError as e: + return e.code, e.headers, e.read() + + +def is_contract_error(headers, raw): + if not (headers.get("Content-Type") or "").startswith("application/json"): + return False + try: + err = json.loads(raw)["error"] + return bool(err["code"] and err["message"] and err["request_id"]) + except (ValueError, KeyError, TypeError): + return False + + +def api_containers(): + ids = subprocess.run(["docker", "compose", "ps", "-a", "-q", "api"], capture_output=True, + text=True, check=True).stdout.split() + out = {} + for cid in ids: + info = json.loads(subprocess.run(["docker", "inspect", cid], capture_output=True, text=True, + check=True).stdout)[0] + out[info["Name"].lstrip("/")] = { + "status": info["State"]["Status"], "oom_killed": info["State"]["OOMKilled"], + "restart_count": info["RestartCount"], "started_at": info["State"]["StartedAt"]} + return out + + +def oom_events(since): + """Имена контейнеров, получивших OOM с момента since (unix time): флаг OOMKilled + сбрасывается после рестарта по restart-policy, поэтому смотрим события Docker.""" + out = subprocess.run(["docker", "events", "--since", str(int(since)), "--until", str(int(time.time()) + 1), + "--filter", "type=container", "--filter", "event=oom", + "--format", "{{.Actor.Attributes.name}}"], capture_output=True, text=True).stdout + return sorted(Counter(out.split()).items()) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--parallel", type=int, default=24) + p.add_argument("--duration", type=float, default=20) + p.add_argument("--output") + a = p.parse_args() + base = a.base_url.rstrip("/") + + username = "flood_" + uuid.uuid4().hex[:10] + password = "correct-horse-" + uuid.uuid4().hex[:8] + st, _, _ = http(base, "POST", "/api/v1/users", + body={"username": username, "display_name": "Flood", "password": password}) + assert st == 201, st + basic = base64.b64encode(("%s:%s" % (username, password)).encode()).decode() + st, _, raw = http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + assert st == 201, st + token = json.loads(raw)["token"] + bad = base64.b64encode(("%s:wrong-password-123" % username).encode()).decode() + + before = api_containers() + wall0 = time.time() + stop = threading.Event() + lock = threading.Lock() + flood, victim = Counter(), Counter() + non_json_errors = Counter() + retry_after_503 = [0, 0] # [503 всего, из них с Retry-After] + + def record(counter, status, headers, raw): + with lock: + counter[str(status)] += 1 + if status >= 400 and not is_contract_error(headers, raw): + non_json_errors[str(status)] += 1 + if status == 503 and counter is flood: + retry_after_503[0] += 1 + retry_after_503[1] += bool(headers.get("Retry-After")) + + def flooder(): + while not stop.is_set(): + try: + record(flood, *http(base, "POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + bad})) + except Exception as exc: # обрыв соединения, timeout + with lock: + flood["exc:" + type(exc).__name__] += 1 + + def prober(): + while not stop.is_set(): + try: + record(victim, *http(base, "GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token})) + except Exception as exc: + with lock: + victim["exc:" + type(exc).__name__] += 1 + time.sleep(0.05) + + threads = [threading.Thread(target=flooder, daemon=True) for _ in range(a.parallel)] + threads.append(threading.Thread(target=prober, daemon=True)) + t0 = time.monotonic() + for t in threads: + t.start() + time.sleep(a.duration) + stop.set() + for t in threads: + t.join(TIMEOUT + 1) + elapsed = time.monotonic() - t0 + time.sleep(3) # дать restart-policy отработать, если реплика упала + after = api_containers() + + restarted = [n for n in after if n in before and ( + after[n]["restart_count"] != before[n]["restart_count"] + or after[n]["started_at"] != before[n]["started_at"])] + victim_ok = victim.get("200", 0) + victim_total = sum(victim.values()) + summary = { + "parallel": a.parallel, "duration_s": round(elapsed, 1), + "flood_statuses": dict(flood), "flood_rps": round(sum(flood.values()) / elapsed, 1), + "flood_503_with_retry_after": "%d/%d" % (retry_after_503[1], retry_after_503[0]), + "victim_statuses": dict(victim), + "victim_error_rate": round(1 - victim_ok / victim_total, 4) if victim_total else None, + "non_contract_error_bodies": dict(non_json_errors), + "containers_before": before, "containers_after": after, + "oom_events": dict(oom_events(wall0)), + "restarted": restarted, + } + summary["pass"] = (not summary["oom_events"] and not restarted and victim_total > 0 + and victim_ok == victim_total and not non_json_errors) + text = json.dumps(summary, ensure_ascii=False, indent=2) + print(text) + if a.output: + with open(a.output, "w", encoding="utf-8") as f: + f.write(text + "\n") + return 0 if summary["pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/legacy/progressive_upload.py b/student-tests/legacy/progressive_upload.py new file mode 100644 index 0000000..1485a28 --- /dev/null +++ b/student-tests/legacy/progressive_upload.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Аудит F02: допустимые по контракту progressive JPEG ~40 Мпикс не должны ронять API. + + docker compose --profile tools run --rm checks /app/student-tests/progressive_upload.py \ + --base-url http://lb:8080 --output /app/evidence/progressive-upload.json + +Файлы маленькие (~0,5–0,9 MB, гладкая картинка), но декодирование progressive JPEG требует +буфер коэффициентов всего кадра: 4:2:0 ~140 MiB, 4:4:4 ~250 MiB (замер на Pillow 13). +Одновременно загружаются 4 progressive 4:4:4 + 1 progressive 4:2:0 + 1 baseline: при двух +репликах за round-robin LB на каждую приходится по 2 тяжёлых декодирования одновременно. Ожидаем: +все 202, все ready, thumbnail 256x256. Что реплики API не были убиты (RestartCount, OOMKilled), +проверяет хост-обёртка api_negative_lab5.py. +""" +import argparse +import concurrent.futures +import io +import json +import sys +import time +from pathlib import Path + +sys.path.insert(0, "/app/tests") +from client import Client, check # noqa: E402 +from PIL import Image # noqa: E402 + +SIDE = 6324 # 39 992 976 пикселей (< 40 000 000) + + +def make(kind, seed): + g = Image.radial_gradient("L").resize((SIDE, SIDE)) + img = Image.merge("RGB", (g, g.rotate(90 * seed), g.transpose(Image.Transpose.FLIP_TOP_BOTTOM))) + kw = {"baseline": {}, "progressive_420": {"progressive": True}, + "progressive_444": {"progressive": True, "subsampling": 0}}[kind] + buf = io.BytesIO() + img.save(buf, "JPEG", quality=85, **kw) + return buf.getvalue() + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://lb:8080") + p.add_argument("--output") + args = p.parse_args() + c = Client(args.base_url, timeout=60) + alice, bob, _eve, chat = c.bootstrap() + files = [("progressive_444", make("progressive_444", i)) for i in range(4)] + \ + [("progressive_420", make("progressive_420", 1)), ("baseline", make("baseline", 0))] + + def up(item): + kind, data = item + t = time.monotonic() + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s.jpg" % (chat["id"], kind), [202, 413, 502, 503, 504], + actor=alice, raw=data, headers={"Content-Type": "image/jpeg"}, schema="Attachment") + return {"kind": kind, "bytes": len(data), "status": r["status"], "seconds": round(time.monotonic() - t, 2), + "instance": r["headers"].get("X-Instance-Id"), + "id": r["json"]["id"] if r["status"] == 202 else None, + "error": r["json"]["error"]["code"] if r["status"] != 202 else None} + + with concurrent.futures.ThreadPoolExecutor(len(files)) as pool: + results = list(pool.map(up, files)) + for r in results: + print(json.dumps(r), flush=True) + for r in results: + check(r["status"] == 202, "%s upload: %s %s" % (r["kind"], r["status"], r["error"])) + for r in results: + att = c.wait_ready(bob, {"id": r["id"]}, wait_seconds=120) + thumb = [v for v in att["variants"] if v["name"] == "thumbnail"][0] + check((thumb["width"], thumb["height"]) == (256, 256), "thumbnail size") + r["ready"] = True + if args.output: + Path(args.output).write_text(json.dumps({"side": SIDE, "pixels": SIDE * SIDE, "uploads": results}, + indent=1) + "\n") + print("PASS: %d concurrent ~40 Mpx JPEG (progressive 4:4:4/4:2:0, baseline) -> 202 -> ready" % len(results)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/legacy/queue_contract_check.py b/student-tests/legacy/queue_contract_check.py new file mode 100644 index 0000000..23fed24 --- /dev/null +++ b/student-tests/legacy/queue_contract_check.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Аудит F12 и F11: разбор сообщения брокера = contracts/jobs.schema.json; версия pipeline. + + python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt rfc3339-validator + .venv/bin/python student-tests/queue_contract_check.py --output evidence/queue-contract-check.json + +F12: набор сообщений (валидные и контрпримеры из аудита) проверяется src/jobs.parse_message +и jsonschema Draft 2020-12 с FormatChecker (date-time, uuid). Вердикты должны совпасть. +F11: API с PIPELINE_VERSION=2 не стартует (понятная ошибка), а не работает «молча» неверно. +""" +import argparse +import copy +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +import jobs # noqa: E402 +from jsonschema import Draft202012Validator, FormatChecker # noqa: E402 + +BASE = {"schema_version": 1, "job_id": "11111111-1111-4111-8111-111111111111", "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", "pipeline_version": 1, "attempt": 1, + "created_at": "2026-09-01T12:00:00Z"} + + +def variant(**kw): + m = copy.deepcopy(BASE) + for k, v in kw.items(): + if v is None: + m.pop(k, None) + else: + m[k] = v + return m + + +CASES = { + "valid minimal": variant(), + "valid with trace context": variant(traceparent="00-" + "a" * 32 + "-" + "b" * 16 + "-01", tracestate="k=v"), + "valid created_at with offset and fraction": variant(created_at="2026-09-01T15:00:00.123+03:00"), + "schema_version true (audit)": variant(schema_version=True), + "created_at date only (audit)": variant(created_at="2026-01-01"), + "traceparent number (audit)": variant(traceparent=123), + "tracestate 513 chars (audit)": variant(tracestate="x" * 513), + "attempt true": variant(attempt=True), + "attempt 0": variant(attempt=0), + "pipeline_version 1.0": variant(pipeline_version=1.0), + "job_id not uuid": variant(job_id="not-a-uuid"), + "job_id uuid without hyphens": variant(job_id="11111111111141118111111111111111"), + "extra field": variant(url="http://evil/"), + "missing created_at": variant(created_at=None), + "wrong type": variant(type="attachment.delete.v1"), + "traceparent bad version": variant(traceparent="01-" + "a" * 32 + "-" + "b" * 16 + "-01"), + "created_at no timezone": variant(created_at="2026-09-01T12:00:00"), +} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--output") + args = p.parse_args() + schema = json.loads((ROOT / "contracts/jobs.schema.json").read_text()) + try: + import rfc3339_validator # noqa: F401 (без него FormatChecker не проверяет date-time) + formats = True + except ImportError: + formats = False + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + results, mismatches = [], 0 + for name, msg in CASES.items(): + schema_ok = validator.is_valid(msg) + try: + jobs.parse_message(json.dumps(msg).encode()) + parser_ok = True + except jobs.InvalidMessage: + parser_ok = False + same = schema_ok == parser_ok + mismatches += not same + results.append({"case": name, "jsonschema_valid": schema_ok, "parser_valid": parser_ok, "match": same}) + print("%s %-45s schema=%-5s parser=%s" % ("PASS" if same else "FAIL", name, schema_ok, parser_ok)) + # F11 + proc = subprocess.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "-e", "PIPELINE_VERSION=2", "api"], + cwd=ROOT, capture_output=True, text=True, timeout=180) + f11 = {"exit_code": proc.returncode, + "message": [l for l in (proc.stdout + proc.stderr).splitlines() if "PIPELINE_VERSION" in l][-1:]} + f11_ok = proc.returncode != 0 and bool(f11["message"]) + print("%s F11 api with PIPELINE_VERSION=2 refuses to start: %s" % ("PASS" if f11_ok else "FAIL", f11)) + if args.output: + Path(args.output).write_text(json.dumps({"date_time_format_checked": formats, "cases": results, + "F11": f11}, ensure_ascii=False, indent=1) + "\n") + ok = mismatches == 0 and f11_ok and formats + print("PASS" if ok else "FAIL", "(%d cases, %d mismatches, date-time format checked: %s)" % (len(CASES), mismatches, formats)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/s3_outage.py b/student-tests/legacy/s3_outage.py new file mode 100644 index 0000000..367fd57 --- /dev/null +++ b/student-tests/legacy/s3_outage.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Недоступность S3 в lab5 (переписано из lab4 по аудиту F14: там ожидался синхронный 201). + + python3 student-tests/s3_outage.py --output evidence/s3-outage.json + +API (как в lab4): при остановленном (`stop`) и зависшем (`pause`) S3 upload и download — быстрый +503 unavailable с Retry-After, текстовые сообщения 201, /health/ready 200, GET метаданных 200. +Worker (новое): задание, принятое ДО отказа S3, при обработке получает storage_unavailable и +уходит в retry с backoff; после восстановления S3 оно доходит до ready. Если отказ длился +дольше всех попыток — failed(retry_exhausted), и оператор делает replay (это тоже проверяется). +""" +import argparse +import json +import time +import uuid + +from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, ops, upload, wait_status, wait_until) + + +def probe(c, alice, chat, att_ready, att_queued): + out = {} + calls = { + "upload": ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"], + {"raw": SAMPLE_PNG, "headers": {"Content-Type": "image/png"}}), + "download_ready": ("GET", "/api/v1/attachments/%s/content" % att_ready["id"], {}), + "metadata_queued": ("GET", "/api/v1/attachments/%s" % att_queued["id"], {}), + "text_message": ("POST", "/api/v1/chats/%s/messages" % chat["id"], + {"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}), + "ready": ("GET", "/health/ready", {}), + } + for name, (method, path, kw) in calls.items(): + t = time.monotonic() + r = c.call(method, path, [200, 201, 202, 503], actor=None if name == "ready" else alice, **kw) + out[name] = {"status": r["status"], "ms": round((time.monotonic() - t) * 1000), + "code": (r["json"] or {}).get("error", {}).get("code"), + "retry_after": r["headers"].get("Retry-After")} + return out + + +def s3_healthy(): + cid = compose("ps", "-q", "s3").stdout.strip() + import subprocess + st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True, + text=True).stdout.strip() + return st == "healthy" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--output") + args = ap.parse_args() + log = Log("S3 outage: API answers 503 fast, accepted jobs retry and complete") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + att_ready = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "before.png")) + res = {} + try: + compose("stop", "worker") + queued = upload(c, alice, chat, SAMPLE_PNG, "accepted-before-outage.png") + log.step("worker stopped, job accepted before outage", attachment_id=queued["id"]) + for action, undo in (("stop", "start"), ("pause", "unpause")): + compose(action, "s3") + time.sleep(5) # S3Health (1 с) заметил отказ, данные устарели (3 с) + pr = probe(c, alice, chat, att_ready, queued) + res["s3_" + action] = pr + log.step("S3 `%s` for 5 s" % action, **pr) + check(pr["upload"]["status"] == 503 and pr["download_ready"]["status"] == 503 + and pr["text_message"]["status"] == 201 and pr["ready"]["status"] == 200 + and pr["metadata_queued"]["status"] == 200, "API during S3 %s: %s" % (action, pr)) + if action == "stop": + compose("up", "-d", "--no-deps", "--no-recreate", "worker") + info = wait_until(lambda: (lambda i: i if "retry_scheduled" in events(i) else None)( + ops("inspect", queued["id"])), 60, 1, "worker retry on storage error") + retries = [e["detail"] for e in info["events"] if e["event"] == "retry_scheduled"] + res["worker_during_outage"] = {"events": events(info), "retries": retries} + log.step("worker meets S3 outage -> retry with backoff", **res["worker_during_outage"]) + check(all(r["error_code"] == "storage_unavailable" for r in retries), "retry reason") + compose(undo, "s3") + t0 = time.monotonic() + wait_until(s3_healthy, 120, 1, "s3 healthy") + log.step("S3 `%s` -> healthy" % undo, seconds=round(time.monotonic() - t0, 1)) + final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 120) + replayed = None + if final["status"] == "failed": + check(final["error_code"] == "retry_exhausted", "unexpected failure " + str(final["error_code"])) + replayed = ops("replay", queued["id"], "--reason", "S3 restored")["replay"][0]["result"] + final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 60) + info = ops("inspect", queued["id"]) + res["accepted_job"] = {"final": final["status"], "replay": replayed, "attempts": info["jobs"][0]["attempt"], + "events": events(info)} + check(final["status"] == "ready", "accepted job not completed") + log.step("job accepted before outage completed", **res["accepted_job"]) + after = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "after.png")) + res["upload_after_recovery"] = after["status"] + log.step("upload after recovery", status=after["status"]) + finally: + compose("unpause", "s3", ok=False) + compose("start", "s3", ok=False) + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", ok=False) + log.save(args.output, results=res) + print("PASS: S3 outage handled by API (503) and worker (retry), accepted work completed") + + +if __name__ == "__main__": + main() diff --git a/student-tests/legacy/s3_private_check.py b/student-tests/legacy/s3_private_check.py new file mode 100644 index 0000000..871154f --- /dev/null +++ b/student-tests/legacy/s3_private_check.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Проверка приватности bucket и ограничений ключей ВНЕ API (внутри backend-сети). + + docker compose --profile tools run --rm checks /app/student-tests/s3_private_check.py + +Проверяет напрямую на S3 (минуя API и его ACL): + * анонимные GET объекта / ListBucket / PUT -> 403; + * ключ api: Put/Get/Delete своего тестового объекта можно; ListBucket, CreateBucket, + DeleteBucket, PutBucketPolicy — AccessDenied; другой bucket — AccessDenied; + * ключ worker (lab5): Get/Put/Delete можно; ListBucket, CreateBucket — AccessDenied; + * ключ janitor: List и Delete можно; GetObject (чтение содержимого) — AccessDenied; + * служебные порты SeaweedFS (master 9333, volume 8080, filer 8888, IAM 8111, + Iceberg 8181, Lance 9101) из backend-сети недоступны — открыт только S3 :8333. +Выход 0 только если все ожидания выполнены. Ключи не печатаются. +""" +import io +import json +import os +import socket +import sys +import urllib.error +import urllib.request +import uuid +from urllib.parse import urlsplit + +sys.path.insert(0, "/app/src") +from minio import Minio # noqa: E402 +from minio.error import S3Error # noqa: E402 + +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") +HOST = urlsplit(ENDPOINT).netloc + + +def key(name): + with open("/run/secrets/" + name) as f: + return f.read().strip() + + +def client(who): + return Minio(HOST, access_key=key("s3_%s_access_key" % who), secret_key=key("s3_%s_secret_key" % who), + secure=False, region=os.environ.get("S3_REGION", "us-east-1")) + + +results = [] + + +def expect(name, should_succeed, fn): + try: + fn() + ok, got = should_succeed, "allowed" + except S3Error as exc: + ok, got = not should_succeed, exc.code + except Exception as exc: # noqa: BLE001 + ok, got = False, type(exc).__name__ + results.append({"check": name, "expected": "allowed" if should_succeed else "denied", "got": got, "pass": ok}) + + +def http_status(method, url, data=None): + req = urllib.request.Request(url, data=data, method=method) + try: + with urllib.request.urlopen(req, timeout=3) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main(): + api, janitor, worker = client("api"), client("janitor"), client("worker") + probe = "originals/s3check-" + uuid.uuid4().hex + expect("api: PutObject в свой bucket", True, lambda: api.put_object(BUCKET, probe, io.BytesIO(b"x" * 16), 16)) + for name, method, url, data in ( + ("anonymous GET object", "GET", "%s/%s/%s" % (ENDPOINT, BUCKET, probe), None), + ("anonymous ListBucket", "GET", "%s/%s?list-type=2" % (ENDPOINT, BUCKET), None), + ("anonymous ListAllMyBuckets", "GET", ENDPOINT + "/", None), + ("anonymous PUT object", "PUT", "%s/%s/anon-%s" % (ENDPOINT, BUCKET, uuid.uuid4().hex), b"z")): + code = http_status(method, url, data) + results.append({"check": name, "expected": "403", "got": code, "pass": code in (401, 403)}) + expect("api: GetObject", True, lambda: api.get_object(BUCKET, probe).read()) + expect("api: ListBucket", False, lambda: list(api.list_objects(BUCKET, recursive=True))) + expect("api: CreateBucket", False, lambda: api.make_bucket("evil-" + uuid.uuid4().hex[:8])) + expect("api: DeleteBucket", False, lambda: api.remove_bucket(BUCKET)) + expect("api: PutBucketPolicy (сделать публичным)", False, lambda: api.set_bucket_policy(BUCKET, json.dumps({ + "Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": "*", "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::%s/*" % BUCKET]}]}))) + expect("janitor: GetObject (чтение содержимого)", False, lambda: janitor.get_object(BUCKET, probe).read()) + expect("janitor: ListBucket", True, lambda: list(janitor.list_objects(BUCKET, prefix="originals/", recursive=True))) + expect("janitor: CreateBucket", False, lambda: janitor.make_bucket("evil-" + uuid.uuid4().hex[:8])) + expect("worker: GetObject (чтение staging)", True, lambda: worker.get_object(BUCKET, probe).read()) + expect("worker: PutObject (результат)", True, lambda: worker.put_object(BUCKET, probe + ".w", io.BytesIO(b"y"), 1)) + expect("worker: DeleteObject (очистка staging)", True, lambda: worker.remove_object(BUCKET, probe + ".w")) + expect("worker: ListBucket", False, lambda: list(worker.list_objects(BUCKET, recursive=True))) + expect("worker: CreateBucket", False, lambda: worker.make_bucket("evil-" + uuid.uuid4().hex[:8])) + expect("api: DeleteObject (компенсация)", True, lambda: api.remove_object(BUCKET, probe)) + s3_host = HOST.split(":")[0] + for port, role in ((9333, "master UI/API"), (8080, "volume"), (8888, "filer UI/API"), (8111, "IAM API"), + (8181, "Iceberg"), (9101, "Lance"), (8333, "S3 API")): + try: + socket.create_connection((s3_host, port), timeout=2).close() + got = "open" + except OSError: + got = "closed" + want = "open" if port == 8333 else "closed" + results.append({"check": "port %d (%s) из backend-сети" % (port, role), "expected": want, "got": got, + "pass": got == want}) + for r in results: + print(("PASS " if r["pass"] else "FAIL ") + "%-45s expected=%-8s got=%s" % (r["check"], r["expected"], r["got"])) + failed = [r for r in results if not r["pass"]] + print(json.dumps({"checks": len(results), "failed": len(failed)})) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/slow_client.py b/student-tests/legacy/slow_client.py new file mode 100644 index 0000000..a242952 --- /dev/null +++ b/student-tests/legacy/slow_client.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Аудит F03: общий срок приёма тела нельзя обойти, присылая данные по байту. + + python3 student-tests/slow_client.py --output evidence/slow-client.json + +На время теста API пересоздаётся с UPLOAD_TIMEOUT_SECONDS=5 и JSON_BODY_TIMEOUT_SECONDS=3 +(чтобы не ждать 60 с), затем возвращается к .env. Сырой сокет через LB: + upload: заголовки + первые 33 байта PNG, дальше по 1 байту каждые 0,5 с (тело 5000 байт — + без ограничения передача длилась бы ~40 мин); + JSON: POST /users с Content-Length 200, по 1 байту каждые 0,5 с. +Ожидаем ответ 400 invalid_request (или закрытие соединения) не позже срок + 2 с, после чего +обычный upload снова 202 (слоты освобождены). +""" +import argparse +import json +import socket +import time +from urllib.parse import urlsplit + +from lab5lib import Client, Log, SAMPLE_PNG, check, recreate, upload + +ENV = {"UPLOAD_TIMEOUT_SECONDS": "5", "JSON_BODY_TIMEOUT_SECONDS": "3"} + + +def drip(base, head, body_first, total_len, limit_s): + u = urlsplit(base) + s = socket.create_connection((u.hostname, u.port or 80), timeout=2) + s.sendall(head + body_first) + started = time.monotonic() + sent = len(body_first) + resp = b"" + while time.monotonic() - started < limit_s: + try: + chunk = s.recv(65536) + if not chunk: + break + resp += chunk + if b"\r\n\r\n" in resp: + break + except socket.timeout: + pass + except OSError: + break + if sent < total_len: + try: + s.sendall(b"\0") + sent += 1 + except OSError: + break + time.sleep(0.5) + elapsed = round(time.monotonic() - started, 2) + s.close() + status = int(resp.split(b" ", 2)[1]) if resp.startswith(b"HTTP/") else None + body = resp.split(b"\r\n\r\n", 1)[1] if b"\r\n\r\n" in resp else b"" + code = None + if body.startswith(b"{"): + try: + code = json.loads(body)["error"]["code"] + except ValueError: + pass + return {"status": status, "code": code, "seconds": elapsed, "bytes_sent": sent, "of": total_len, + "x_instance_id": b"x-instance-id" in resp.lower()} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--output") + args = p.parse_args() + log = Log("slow client vs total body deadline") + recreate("api", ENV) + log.step("api recreated", env=ENV) + res = {} + try: + c = Client(args.base_url, timeout=30) + alice, _bob, _eve, chat = c.bootstrap() + u = urlsplit(args.base_url) + head = ("POST /api/v1/chats/%s/attachments?filename=slow.png HTTP/1.1\r\nHost: %s\r\n" + "Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: 5000\r\n\r\n" + % (chat["id"], u.netloc, alice["token"])).encode() + res["upload"] = drip(args.base_url, head, SAMPLE_PNG[:33], 5000, 20) + log.step("upload drip 1 B / 0.5 s", **res["upload"]) + check(res["upload"]["seconds"] <= 7 and res["upload"]["bytes_sent"] < 5000, "upload deadline not enforced") + check(res["upload"]["status"] == 400 and res["upload"]["x_instance_id"], "upload: expected API 400") + head = ("POST /api/v1/users HTTP/1.1\r\nHost: %s\r\nContent-Type: application/json\r\n" + "Content-Length: 200\r\n\r\n" % u.netloc).encode() + res["json"] = drip(args.base_url, head, b'{"username":', 200, 20) + log.step("json drip 1 B / 0.5 s", **res["json"]) + check(res["json"]["seconds"] <= 5 and res["json"]["bytes_sent"] < 200, "json deadline not enforced") + check(res["json"]["status"] == 400 and res["json"]["x_instance_id"], "json: expected API 400") + att = upload(c, alice, chat, SAMPLE_PNG, "after-slow.png") + res["normal_upload_after"] = att["status"] + log.step("normal upload afterwards", status=att["status"]) + finally: + recreate("api") + log.step("api restored to .env configuration") + log.save(args.output, results=res) + print("PASS: slow clients are cut off by the total deadline") + + +if __name__ == "__main__": + main() diff --git a/student-tests/legacy/upload_memory.py b/student-tests/legacy/upload_memory.py new file mode 100644 index 0000000..29845f5 --- /dev/null +++ b/student-tests/legacy/upload_memory.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Пик памяти API при одновременных загрузках крупных файлов (уровень 4). + + python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json + +Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT +частями по 5 MiB и проверка JPEG в подпроцессе с лимитом (src/jpeg_probe.py) держат RAM +реплики ограниченной и не зависящей линейно от числа одновременных загрузок. + +Lab5 (аудит F16): явные критерии успеха и ненулевой код выхода при нарушении — все ответы 202 +(503 только с --allow-503), ни одного OOM-kill и перезапуска реплик, пик памяти cgroup каждой +реплики < 90% лимита. Объём считается по размеру файла в каждой записи. В смесь входят +несжимаемый PNG ~10 MiB, baseline JPEG и progressive JPEG 4:4:4 ~40 Мпикс (самый тяжёлый +по памяти вход, аудит F02) — JPEG генерируются в контейнере checks (Pillow). + +Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой +реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и +anon/file из cgroup memory.stat. Нагрузка — concurrency потоков, каждый грузит свой +файл через LB; файлы: несжимаемый PNG ~9.99 MiB и (если на хосте есть Pillow) JPEG +6000x6000 = 36 Мпикс. В конце — VmHWM (пиковый RSS процесса) и memory.peak cgroup. +file-память cgroup — page cache временных файлов, она вытесняема и OOM не вызывает. +""" +import argparse +import hashlib +import io +import json +import os +import subprocess +import sys +import threading +import time +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "student-tests")) +from client import Client # noqa: E402 +from attachments_negative import noise_png # noqa: E402 + +SAMPLER = ("while true; do r=$(grep VmRSS /proc/1/status | tr -s ' ' | cut -d' ' -f2); " + "a=$(grep '^anon ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); " + "f=$(grep '^file ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); echo \"$r $a $f\"; sleep 0.05; done") + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=True).stdout + + +def containers(): + return sh("docker", "compose", "ps", "-q", "api").split() + + +def proc_status(cid, field): + out = sh("docker", "exec", cid, "grep", field, "/proc/1/status") + return int(out.split()[1]) // 1024 # kB -> MiB + + +class Sampler: + def __init__(self, cid): + self.cid, self.samples = cid, [] + self.p = subprocess.Popen(["docker", "exec", cid, "sh", "-c", SAMPLER], stdout=subprocess.PIPE, text=True) + threading.Thread(target=self._read, daemon=True).start() + + def _read(self): + for line in self.p.stdout: + parts = line.split() + if len(parts) == 3: + self.samples.append((time.monotonic(), int(parts[0]) / 1024, int(parts[1]) / 2**20, int(parts[2]) / 2**20)) + + def stop(self): + self.p.terminate() + + +GEN = r""" +import io, sys +from PIL import Image +kind = sys.argv[1] +if kind == "baseline": + im = Image.effect_noise((6000, 6000), 40).convert("RGB") + for q in (60, 45, 30, 20): + b = io.BytesIO(); im.save(b, "JPEG", quality=q) + if b.tell() <= 10 * 1024 * 1024: + break +else: + g = Image.radial_gradient("L").resize((6324, 6324)) + b = io.BytesIO(); Image.merge("RGB", (g, g.rotate(90), g)).save(b, "JPEG", quality=85, progressive=True, subsampling=0) +sys.stdout.buffer.write(b.getvalue()) +""" + + +def make_jpeg(kind): + """JPEG генерируется в контейнере checks (на хосте Pillow может не быть).""" + proc = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "checks", "-c", GEN, kind], + cwd=ROOT, capture_output=True, check=True) + return proc.stdout + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--concurrency", type=int, default=12) + ap.add_argument("--rounds", type=int, default=3) + ap.add_argument("--no-restart", action="store_true") + ap.add_argument("--allow-503", action="store_true", help="считать 503 (перегрузка) допустимым ответом") + ap.add_argument("--output") + args = ap.parse_args() + files = [("image/png", noise_png(10 * 1024 * 1024))] + files.append(("image/jpeg", make_jpeg("baseline"))) + files.append(("image/jpeg", make_jpeg("progressive444"))) + if not args.no_restart: + sh("docker", "compose", "restart", "api") + sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api") + time.sleep(4) + c = Client(args.base_url, timeout=120) + alice, _bob, _eve, chat = c.bootstrap() + # прогрев: один upload каждого типа (импорт Pillow, пулы соединений) + for ctype, data in files: + c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data, + headers={"Content-Type": ctype}) + cids = containers() + restarts_before = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split() + idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")} + for cid in cids} + s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0] + s3_before = {"rss_mib": proc_status(s3_cid, "VmRSS"), "oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip()} + samplers = [Sampler(cid) for cid in cids] + s3_sampler = Sampler(s3_cid) + time.sleep(1) + results, lock = [], threading.Lock() + + def worker(i): + ctype, data = files[i % len(files)] + t = time.monotonic() + try: + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=m%d" % (chat["id"], i), [202, 503], actor=alice, + raw=data, headers={"Content-Type": ctype}, schema=None) + st, inst = r["status"], r["headers"].get("X-Instance-Id") + except Exception as exc: # noqa: BLE001 + st, inst = type(exc).__name__, None + with lock: + # Размер — в самой записи: потоки завершаются не по порядку (аудит F16). + results.append({"type": ctype, "bytes": len(data), "status": st, "instance": inst, + "s": round(time.monotonic() - t, 2)}) + + started = time.monotonic() + for _ in range(args.rounds): + threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)] + for th in threads: + th.start() + for th in threads: + th.join() + wall = time.monotonic() - started + time.sleep(1) + for s in samplers + [s3_sampler]: + s.stop() + per = {} + for cid, s in zip(cids, samplers): + k = cid[:12] + per[k] = dict(idle[k], + samples=len(s.samples), + peak_rss_mib_sampled=round(max(x[1] for x in s.samples), 1), + peak_anon_mib=round(max(x[2] for x in s.samples), 1), + peak_file_cache_mib=round(max(x[3] for x in s.samples), 1), + vmhwm_mib=proc_status(cid, "VmHWM"), + cgroup_memory_peak_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak")) / 2**20, 1), + memory_limit_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20), + uploads_served=sum(1 for r in results if r["instance"] == k)) + durations = sorted(r["s"] for r in results if r["status"] == 202) + summary = { + "concurrency": args.concurrency, "rounds": args.rounds, + "files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files], + "uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k) + for k in {r["status"] for r in results}}, + "megabytes_uploaded_ok": round(sum(r["bytes"] for r in results if r["status"] == 202) / 2**20, 1), + "wall_s": round(wall, 1), + "latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None, + "replicas": per, + "s3": dict(s3_before, + peak_rss_mib_sampled=round(max(x[1] for x in s3_sampler.samples), 1) if s3_sampler.samples else None, + peak_anon_mib=round(max(x[2] for x in s3_sampler.samples), 1) if s3_sampler.samples else None, + restarts=int(sh("docker", "inspect", "-f", "{{.RestartCount}}", s3_cid)), + oom_killed_after=sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip(), + memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)), + "oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(), + } + allowed = {202, 503} if args.allow_503 else {202} + restarts_after = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split() + criteria = { + "all_responses_allowed": all(r["status"] in allowed for r in results), + "no_oom_kills": all(x == "false" for x in summary["oom_kills"]), + "no_replica_restarts": restarts_before == restarts_after, + "cgroup_peak_below_90pct": all(v["cgroup_memory_peak_mib"] < 0.9 * v["memory_limit_mib"] for v in per.values()), + } + summary["criteria"] = criteria + summary["pass"] = all(criteria.values()) + print(json.dumps(summary, ensure_ascii=False, indent=1)) + if args.output: + Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1)) + return 0 if summary["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/legacy/variants_check.py b/student-tests/legacy/variants_check.py new file mode 100644 index 0000000..2a75ddc --- /dev/null +++ b/student-tests/legacy/variants_check.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Проверка вариантов по контракту (запускается В КОНТЕЙНЕРЕ checks, где есть Pillow): + + docker compose --profile tools run --rm checks /app/student-tests/variants_check.py \ + --base-url http://lb:8080 --output /app/evidence/variants-check.json + +Для каждого входа: ready, ровно 3 варианта, метаданные = фактическим байтам (размер, SHA-256, +декодированные размеры, формат), оригинал неизменен. Thumbnail: JPEG, вписан в 256x256 без +увеличения, округление вниз; EXIF orientation применена; alpha -> белый. Crop: JPEG 128x128. +Optimized: WebP, вписан в 1280x1280 без увеличения. В вариантах нет метаданных — проверяются +СЫРЫЕ байты (аудит F07): в JPEG нет маркеров COM, APP1 (Exif/XMP), APP2 (ICC); в WebP нет +чанков EXIF, XMP, ICCP. Прозрачность tRNS у RGB и L (аудит F06) тоже даёт белый фон в JPEG +и alpha=0 в WebP. +""" +import argparse +import hashlib +import io +import json +import sys +import time +from pathlib import Path + +sys.path.insert(0, "/app/tests") +from client import Client, check # noqa: E402 +from PIL import Image # noqa: E402 + +FIXTURE = Path("/app/tests/fixtures/sample.png").read_bytes() + + +def fit(w, h, box): + if w <= box and h <= box: + return w, h + return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box) + + +def encode(img, fmt, **kw): + buf = io.BytesIO() + img.save(buf, fmt, **kw) + return buf.getvalue() + + +def jpeg_markers(b): + """Маркеры сегментов JPEG до начала скана (SOS).""" + out, i = [], 2 + while i + 4 <= len(b) and b[i] == 0xFF: + m = b[i + 1] + if m == 0xDA: + break + out.append(m) + i += 2 + int.from_bytes(b[i + 2:i + 4], "big") + return out + + +def webp_chunks(b): + out, i = [], 12 + while i + 8 <= len(b): + name, size = b[i:i + 4], int.from_bytes(b[i + 4:i + 8], "little") + out.append(name.decode("latin-1")) + i += 8 + size + (size & 1) + return out + + +def no_metadata(b, fmt): + if fmt == "JPEG": + bad = {0xFE: "COM", 0xE1: "APP1", 0xE2: "APP2"} + found = [bad[m] for m in jpeg_markers(b) if m in bad] + else: + found = [c for c in webp_chunks(b) if c.strip() in ("EXIF", "XMP", "ICCP")] + return found + + +def cases(): + out = [("fixture 320x200 PNG", FIXTURE, "image/png", (320, 200), None)] + # RGBA: левая половина прозрачная -> в JPEG должна стать белой. + rgba = Image.new("RGBA", (300, 300), (200, 20, 20, 255)) + rgba.paste((0, 0, 0, 0), (0, 0, 150, 300)) + out.append(("RGBA 300x300 half transparent", encode(rgba, "PNG"), "image/png", (300, 300), "alpha")) + # EXIF orientation 6 (поворот 90° по часовой): хранится 400x200, показывается 200x400. + # Верхняя половина ХРАНИМОГО кадра красная -> после поворота красной становится правая половина. + raw = Image.new("RGB", (400, 200), (0, 0, 255)) + raw.paste((255, 0, 0), (0, 0, 400, 100)) + exif = Image.Exif() + exif[0x0112] = 6 + exif[0x010F] = "MTUSI test camera" # Make: проверяем, что EXIF не утёк в варианты + out.append(("JPEG 400x200 EXIF orientation=6", encode(raw, "JPEG", quality=95, exif=exif.tobytes()), + "image/jpeg", (200, 400), "orientation")) + # Все виды метаданных сразу: комментарий, EXIF, XMP, ICC (аудит F07). + from PIL import ImageCms + icc = ImageCms.ImageCmsProfile(ImageCms.createProfile("sRGB")).tobytes() + meta = Image.new("RGB", (500, 300), (30, 90, 160)) + meta_exif = Image.Exif() + meta_exif[0x010F] = "PRIVATE-CAMERA" # Make, без orientation + out.append(("JPEG with COM+EXIF+XMP+ICC", encode(meta, "JPEG", quality=90, comment=b"PRIVATE-COMMENT", + exif=meta_exif.tobytes(), icc_profile=icc, + xmp=b"PRIVATE-XMP"), + "image/jpeg", (500, 300), "metadata")) + # tRNS у RGB и L (аудит F06): весь кадр — «прозрачный цвет». + out.append(("RGB PNG with tRNS (fully transparent)", encode(Image.new("RGB", (300, 200), (255, 0, 0)), "PNG", + transparency=(255, 0, 0)), "image/png", (300, 200), "alpha_full")) + out.append(("L PNG with tRNS (fully transparent)", encode(Image.new("L", (200, 300), 0), "PNG", transparency=0), + "image/png", (200, 300), "alpha_full")) + out.append(("tiny PNG 10x5 (no upscale)", encode(Image.new("RGB", (10, 5), (9, 200, 9)), "PNG"), + "image/png", (10, 5), None)) + big = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB") + out.append(("JPEG 3000x2000", encode(big, "JPEG", quality=90), "image/jpeg", (3000, 2000), None)) + tall = Image.new("RGB", (100, 1000), (50, 60, 70)) + out.append(("tall PNG 100x1000", encode(tall, "PNG"), "image/png", (100, 1000), None)) + pal = Image.new("P", (64, 64), 1) + pal.putpalette([255, 255, 255, 10, 200, 10] + [0] * 762) + out.append(("palette PNG with transparency", encode(pal, "PNG", transparency=1), "image/png", (64, 64), "alpha")) + return out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://lb:8080") + p.add_argument("--output") + args = p.parse_args() + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + results = [] + pending = [] + for name, data, ctype, (ow, oh), special in cases(): + att = c.call("POST", "/api/v1/chats/%s/attachments?filename=v.%s" % (chat["id"], ctype.split("/")[1]), 202, + actor=alice, raw=data, headers={"Content-Type": ctype}, schema="Attachment")["json"] + pending.append((name, data, ctype, ow, oh, special, att["id"])) + for name, data, ctype, ow, oh, special, att_id in pending: + deadline = time.monotonic() + 60 + while True: + att = c.call("GET", "/api/v1/attachments/" + att_id, actor=bob, schema="Attachment")["json"] + if att["status"] in ("ready", "failed") or time.monotonic() > deadline: + break + time.sleep(0.3) + check(att["status"] == "ready", "%s: status %s" % (name, att["status"])) + r = {"case": name, "source_oriented": [ow, oh], "variants": {}} + orig = c.call("GET", "/api/v1/attachments/%s/content" % att_id, actor=bob) + check(orig["bytes"] == data and orig["headers"].get_content_type() == ctype, name + ": original changed") + check(sorted(v["name"] for v in att["variants"]) == ["crop", "optimized", "thumbnail"], name + ": variants") + expect = {"thumbnail": ("image/jpeg", "JPEG", fit(ow, oh, 256)), + "crop": ("image/jpeg", "JPEG", (128, 128)), + "optimized": ("image/webp", "WEBP", fit(ow, oh, 1280))} + for v in att["variants"]: + ct, fmt, size = expect[v["name"]] + body = c.call("GET", "/api/v1/attachments/%s/content?variant=%s" % (att_id, v["name"]), actor=bob) + b = body["bytes"] + img = Image.open(io.BytesIO(b)) + img.load() + check(v["content_type"] == ct and body["headers"].get_content_type() == ct, name + ": content type") + check(img.format == fmt, name + ": real format " + str(img.format)) + check((v["width"], v["height"]) == size == img.size, "%s %s: size %s meta %s expected %s" % ( + name, v["name"], img.size, (v["width"], v["height"]), size)) + check(v["size_bytes"] == len(b) and v["sha256"] == hashlib.sha256(b).hexdigest(), name + ": checksum") + leaked = no_metadata(b, fmt) + check(not leaked and not img.getexif() and "icc_profile" not in img.info, + "%s: metadata leaked into %s: %s" % (name, v["name"], leaked)) + check(b"PRIVATE" not in b, name + ": private text found in bytes of " + v["name"]) + item = {"size": list(img.size), "format": fmt, "bytes": len(b), "metadata_markers": leaked} + if special == "alpha_full": + if v["name"] == "optimized": + a = img.convert("RGBA").getpixel((img.size[0] // 2, img.size[1] // 2))[3] + check(a == 0, name + ": WebP alpha must be 0, got %d" % a) + item["webp_alpha"] = a + else: + px = img.convert("RGB").getpixel((img.size[0] // 2, img.size[1] // 2)) + check(min(px) >= 245, name + ": transparent area is not white: %s" % (px,)) + item["center_rgb"] = list(px) + if special == "alpha" and v["name"] in ("thumbnail", "crop"): + px = img.convert("RGB").getpixel((1, img.size[1] // 2)) + check(min(px) >= 245, name + ": transparent area is not white: %s" % (px,)) + item["transparent_pixel_rgb"] = list(px) + if special == "orientation" and v["name"] == "thumbnail": + w, h = img.size + left, right = img.getpixel((w // 4, h // 2)), img.getpixel((3 * w // 4, h // 2)) + check(right[0] > 200 and right[2] < 60 and left[2] > 200, name + ": orientation not applied") + item["left_rgb"], item["right_rgb"] = list(left), list(right) + r["variants"][v["name"]] = item + results.append(r) + print("PASS %-36s %s" % (name, {k: tuple(x["size"]) for k, x in r["variants"].items()}), flush=True) + c.call("GET", "/api/v1/attachments/%s/content?variant=bogus" % pending[0][6], 400, actor=bob) + if args.output: + Path(args.output).write_text(json.dumps({"cases": results, "unknown_variant_status": 400}, + ensure_ascii=False, indent=1) + "\n") + print("PASS: %d inputs, all variants match contract" % len(results)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/obs_correlation.py b/student-tests/obs_correlation.py new file mode 100644 index 0000000..2115c33 --- /dev/null +++ b/student-tests/obs_correlation.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Lab6, уровни 3 и 4: корреляция сигналов на контролируемых запросах (всё через Grafana). + + python3 student-tests/obs_correlation.py --output evidence/obs-correlation.json [--retry] + +1. Сообщение: по X-Trace-Id ответа найден trace в Tempo (SERVER span маршрута-шаблона + db spans), + по X-Request-Id — структурный лог event=http.request в Loki с тем же trace_id/span_id. +2. Upload -> ready: ОДИН trace связывает API (POST .../attachments, db, s3 PutObject staging, + PRODUCER «create») -> relay (PRODUCER «attachments send», parent = create) -> worker (CONSUMER + «process», parent = send; db jobs.claim, fetch/transform, s3 PutObject x4, db jobs.complete, + s3 DeleteObject). traceparent в outbox совпадает с trace. Лог worker job.completed — с тем же trace_id. +3. --retry: worker пересоздаётся с WORKER_FAULT_INJECTION=1; fault-transient-1.png — 1-я попытка + получает временную ошибку (retry_scheduled), 2-я — ready; обе попытки и повторная публикация + relay находятся в том же trace. После теста worker возвращается в обычный режим. +4. Утечки: пароль, токен, текст сообщения и filename не встречаются ни в Loki, ни в трассах; + у метрик приложений нет labels user_id/chat_id/... и значений-UUID. +""" +import argparse +import json +import re +import sys +import time +import uuid +from pathlib import Path + +from lab5lib import SAMPLE_PNG, Client, Failure, Log, check, ops, recreate, wait_status +import obslib + +FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "job_id", "url", + "url_full", "url_path", "filename", "text", "trace_id", "span_id"} +UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +def by_name(spans, name, service=None): + return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)] + + +def one(spans, name, service=None): + found = by_name(spans, name, service) + check(found, "span %r (%s) not found" % (name, service)) + return found[0] + + +def loki_lines(query, t0, retries=15): + for _ in range(retries): + lines = obslib.loki_query(query, t0 - 30, time.time() + 5, limit=200) + if lines: + return lines + time.sleep(2) + return [] + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + p.add_argument("--retry", action="store_true", help="also prove retry chain (recreates worker)") + args = p.parse_args() + log = Log("lab6 signal correlation: request -> trace -> log, upload -> relay -> worker") + c = Client(args.base_url, timeout=30) + t0 = time.time() + alice, bob, eve, chat = c.bootstrap() + marker = "lab6-marker-" + uuid.uuid4().hex + leak_strings = {"message_text": marker, "password": alice["registration"]["password"], + "token": alice["token"]} + result = {} + + # ---------------------------------------------------------------- 1. сообщение + r = c.message(alice, chat, marker) + rid, tid, inst = r["headers"]["X-Request-Id"], r["headers"].get("X-Trace-Id"), r["headers"]["X-Instance-Id"] + check(tid and re.fullmatch(r"[0-9a-f]{32}", tid), "X-Trace-Id header missing") + log.step("message sent", request_id=rid, trace_id=tid, instance=inst) + spans = obslib.wait_trace(tid, lambda s: by_name(s, "db add_message")) + server = one(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api") + db = one(spans, "db add_message") + check(server["kind"] == "SPAN_KIND_SERVER" and db["kind"] == "SPAN_KIND_CLIENT", "span kinds") + check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span") + check(server["attrs"].get("http.response.status_code") in (201, "201"), "status code in span") + check(server["instance"] == inst, "span instance = X-Instance-Id") + log.step("message trace in Tempo", spans=len(spans), server_ms=server["duration_ms"], + db_spans=[s["name"] for s in spans if s["name"].startswith("db ")]) + result["message_trace_tree"] = obslib.tree(spans) + lines = loki_lines('{service_name="messenger-api"} | request_id="%s"' % rid, t0) + check(lines, "API log with request_id not found in Loki") + rec = lines[-1] + check(rec["labels"].get("trace_id") == tid, "log trace_id differs from X-Trace-Id") + check(rec["labels"].get("span_id") == server["span_id"], "log span_id differs from server span") + check(rec["labels"].get("event") == "http.request", "log event") + log.step("structured API log in Loki", event=rec["labels"].get("event"), trace_id=rec["labels"].get("trace_id"), + span_id=rec["labels"].get("span_id"), severity=rec["labels"].get("severity_text"), + service_name=rec["labels"].get("service_name"), line=rec["line"][:120]) + result["message"] = {"request_id": rid, "trace_id": tid, "loki_labels": sorted(rec["labels"])} + + # ---------------------------------------------------------------- 2. upload -> worker + fname = "leakcheck-%s.png" % uuid.uuid4().hex[:10] + leak_strings["filename"] = fname + resp = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], fname), 202, actor=alice, + raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment") + att, utid = resp["json"], resp["headers"]["X-Trace-Id"] + ready = wait_status(c, alice, att["id"], ("ready",), timeout=60) + log.step("upload accepted and ready", attachment_id=att["id"], trace_id=utid, status=ready["status"]) + need = ["POST /api/v1/chats/{chat_id}/attachments", "attachment.process.v1 create", "attachments send", + "attachments.process.v1 process", "db jobs.complete"] + spans = obslib.wait_trace(utid, lambda s: all(by_name(s, n) for n in need) and by_name(s, "s3 DeleteObject"), + timeout=60) + srv = one(spans, need[0], "messenger-api") + create = one(spans, need[1], "messenger-api") + send = one(spans, need[2], "messenger-relay") + proc = one(spans, need[3], "messenger-worker") + check(create["kind"] == "SPAN_KIND_PRODUCER" and send["kind"] == "SPAN_KIND_PRODUCER", "producer kinds") + check(proc["kind"] == "SPAN_KIND_CONSUMER", "consumer kind") + check(send["parent_id"] == create["span_id"], "relay send span is child of API create span") + check(proc["parent_id"] == send["span_id"], "worker process span is child of relay send span") + puts = [s for s in by_name(spans, "s3 PutObject") if s["service"] == "messenger-worker"] + check(len(puts) == 4, "worker wrote original + 3 variants (got %d)" % len(puts)) + check(by_name(spans, "s3 PutObject", "messenger-api"), "API staging PUT span") + services = sorted({s["service"] for s in spans}) + check(services == ["messenger-api", "messenger-relay", "messenger-worker"], "services in trace %s" % services) + info = ops("inspect", att["id"]) + job_id = info["jobs"][0]["id"] + tps = [o.get("traceparent") for o in info["outbox"]] + check(all(tp and tp.split("-")[1] == utid for tp in tps), "outbox traceparent belongs to upload trace") + gap_ms = round((proc["start"] - srv["end"]) * 1000, 1) + log.step("one trace upload -> relay -> worker", spans=len(spans), services=services, job_id=job_id, + outbox_traceparent=tps[0], process_parent_is_send=True, send_parent_is_create=True, + worker_s3_puts=len(puts), api_to_worker_gap_ms=gap_ms, + process_ms=proc["duration_ms"], outcome=proc["attrs"].get("messenger.job.outcome")) + result["upload_trace_tree"] = obslib.tree(spans) + wl = loki_lines('{service_name="messenger-worker"} | trace_id="%s"' % utid, t0) + events = [w["labels"].get("event") for w in wl] + check("job.claimed" in events and "job.completed" in events, "worker logs for job: %s" % events) + comp = [w for w in wl if w["labels"].get("event") == "job.completed"][0] + check(comp["labels"].get("job_id") == job_id, "job_id in worker log") + log.step("worker structured logs in Loki by trace_id", events=events, job_id=comp["labels"].get("job_id"), + span_id=comp["labels"].get("span_id"), line=comp["line"][:120]) + result["upload"] = {"attachment_id": att["id"], "job_id": job_id, "trace_id": utid, "worker_log_events": events} + + # ---------------------------------------------------------------- 3. retry в той же трассе + if args.retry: + env = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "4"} + recreate("worker", env) + try: + resp = c.call("POST", "/api/v1/chats/%s/attachments?filename=fault-transient-1.png" % chat["id"], 202, + actor=alice, raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment") + ratt, rtid = resp["json"], resp["headers"]["X-Trace-Id"] + wait_status(c, alice, ratt["id"], ("ready",), timeout=90) + spans = obslib.wait_trace(rtid, lambda s: len(by_name(s, "attachments.process.v1 process")) >= 2 + and by_name(s, "db jobs.complete"), timeout=60) + procs = by_name(spans, "attachments.process.v1 process") + sends = by_name(spans, "attachments send") + creates = by_name(spans, "attachment.process.v1 create") + outcomes = [s["attrs"].get("messenger.job.outcome") for s in procs] + check(len(procs) == 2 and outcomes == ["retry", "ready"], "attempt outcomes %s" % outcomes) + check(len(sends) == 2 and len(creates) == 2, "two publications (initial + retry)") + retry_create = [s for s in creates if s["service"] == "messenger-worker"] + check(retry_create and retry_create[0]["parent_id"] in {x["span_id"] for x in spans}, "retry create in worker") + check(sends[1]["parent_id"] == retry_create[0]["span_id"], "second send is child of worker retry create") + check(procs[1]["parent_id"] == sends[1]["span_id"], "second attempt consumes second publication") + log.step("retry chain in one trace", trace_id=rtid, attempts=len(procs), outcomes=outcomes, + retry_delay_s=round(procs[1]["start"] - procs[0]["end"], 2), + error_type=procs[0]["attrs"].get("error.type")) + result["retry_trace_tree"] = obslib.tree(spans) + result["retry"] = {"trace_id": rtid, "outcomes": outcomes} + finally: + recreate("worker") + log.step("worker restored without fault injection") + + # ---------------------------------------------------------------- 4. утечки + time.sleep(3) + leaks = {} + for name, value in leak_strings.items(): + hits = obslib.loki_query('{service_name=~"messenger-.+"} |= `%s`' % value, t0 - 30, time.time() + 5, limit=5) + leaks["loki:" + name] = len(hits) + blobs = json.dumps([obslib.tempo_trace(t) for t in (tid, utid)], ensure_ascii=False) + for name, value in leak_strings.items(): + leaks["tempo:" + name] = blobs.count(value) + check(all(v == 0 for v in leaks.values()), "telemetry leak: %s" % leaks) + labels = set(obslib.prom("labels", **{"match[]": '{job=~"messenger-.+"}'})["data"]) + bad_labels = sorted(labels & FORBIDDEN_LABELS) + uuid_values = {} + for lab in sorted(labels - {"__name__"}): + vals = obslib.prom("label/%s/values" % lab, **{"match[]": '{job=~"messenger-.+"}'})["data"] + u = [v for v in vals if UUID_RE.search(v)] + if u: + uuid_values[lab] = u[:3] + check(not bad_labels and not uuid_values, "high-cardinality labels: %s %s" % (bad_labels, uuid_values)) + log.step("no secrets/bodies in telemetry, no ID labels in metrics", leaks=leaks, metric_labels=sorted(labels)) + result["leaks"] = leaks + result["metric_labels"] = sorted(labels) + log.save(args.output, **result) + if args.output: + Path(args.output).with_suffix(".trees.txt").write_text( + "\n".join(["# message trace %s" % tid] + result["message_trace_tree"] + ["", "# upload trace %s" % utid] + + result["upload_trace_tree"] + (["", "# retry trace %s" % result["retry"]["trace_id"]] + + result["retry_trace_tree"] if args.retry else [])) + "\n", + encoding="utf-8") + print("PASS: correlation request->trace->log and upload->relay->worker%s" % (" incl. retry" if args.retry else "")) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + sys.exit(1) diff --git a/student-tests/obslib.py b/student-tests/obslib.py new file mode 100644 index 0000000..e5a850a --- /dev/null +++ b/student-tests/obslib.py @@ -0,0 +1,160 @@ +"""Помощники lab6: запросы к Prometheus / Loki / Tempo ЧЕРЕЗ Grafana (datasource proxy). + +Так проверка заодно подтверждает, что Grafana видит все три хранилища по provisioning. +Пароль admin Grafana читается из secrets/grafana_admin_password и нигде не печатается. +Только стандартная библиотека. +""" +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +GRAFANA = os.environ.get("GRAFANA_URL", "https://localhost:%s" % os.environ.get("GRAFANA_PORT", "3000")) +# Lab7: Grafana — HTTPS с сертификатом public-ca (SAN localhost/127.0.0.1); проверка обязательна. +import ssl # noqa: E402 +_CTX = ssl.create_default_context(cafile=os.environ.get("CA_FILE") or str(ROOT / "secrets/pki/public-ca.crt")) + + +def _auth_header(): + password = (ROOT / "secrets/grafana_admin_password").read_text(encoding="utf-8").strip() + return "Basic " + base64.b64encode(("admin:" + password).encode()).decode() + + +def grafana(path, params=None, timeout=20, accept="application/json"): + url = GRAFANA + path + ("?" + urllib.parse.urlencode(params, doseq=True) if params else "") + req = urllib.request.Request(url, headers={"Authorization": _auth_header(), "Accept": accept}) + try: + with urllib.request.urlopen(req, timeout=timeout, context=_CTX) as resp: + return json.loads(resp.read() or b"null") + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + raise RuntimeError("grafana %s -> HTTP %d: %s" % (path, exc.code, exc.read()[:300])) + + +# ------------------------------------------------------------------ Prometheus +def prom(path, **params): + return grafana("/api/datasources/proxy/uid/prometheus/api/v1/" + path, params) + + +def prom_query(expr, at=None): + params = {"query": expr} + if at is not None: + params["time"] = "%.3f" % at + data = prom("query", **params) + return data["data"]["result"] + + +def prom_value(expr, at=None, default=None): + res = prom_query(expr, at) + if not res: + return default + return float(res[0]["value"][1]) + + +def prom_range(expr, start, end, step=15): + data = prom("query_range", query=expr, start="%.3f" % start, end="%.3f" % end, step=str(step)) + return data["data"]["result"] + + +def prom_alerts(): + return prom("alerts")["data"]["alerts"] + + +# ------------------------------------------------------------------ Loki +def loki_query(query, start, end, limit=100, direction="forward"): + data = grafana("/api/datasources/proxy/uid/loki/loki/api/v1/query_range", + {"query": query, "start": str(int(start * 1e9)), "end": str(int(end * 1e9)), + "limit": str(limit), "direction": direction}) + out = [] + for stream in data["data"]["result"]: + for ts, line, *meta in stream["values"]: + item = {"ts": int(ts) / 1e9, "line": line, "labels": stream["stream"]} + if meta: + item["metadata"] = meta[0] + out.append(item) + out.sort(key=lambda x: x["ts"]) + return out + + +# ------------------------------------------------------------------ Tempo +def tempo_trace(trace_id): + """Трасса в формате OTLP JSON (batches/resourceSpans) или None, если ещё не доступна.""" + return grafana("/api/datasources/proxy/uid/tempo/api/traces/" + trace_id) + + +def tempo_search(traceql, start, end, limit=20): + return grafana("/api/datasources/proxy/uid/tempo/api/search", + {"q": traceql, "start": str(int(start)), "end": str(int(end)), "limit": str(limit)}) + + +def _attr_value(v): + for key in ("stringValue", "intValue", "boolValue", "doubleValue"): + if key in v: + return v[key] + return v + + +def _hex(b64_or_hex): + if not b64_or_hex: + return "" + if len(b64_or_hex) in (16, 32) and all(c in "0123456789abcdef" for c in b64_or_hex): + return b64_or_hex + return base64.b64decode(b64_or_hex).hex() + + +def flatten_spans(trace): + """[{service, name, kind, span_id, parent_id, start, end, duration_ms, status, attrs}]""" + spans = [] + batches = (trace or {}).get("batches") or (trace or {}).get("resourceSpans") or [] + for batch in batches: + res = {a["key"]: _attr_value(a["value"]) for a in batch.get("resource", {}).get("attributes", [])} + for scope in batch.get("scopeSpans") or batch.get("instrumentationLibrarySpans") or []: + for s in scope.get("spans", []): + start, end = int(s["startTimeUnixNano"]), int(s["endTimeUnixNano"]) + spans.append({ + "service": res.get("service.name"), "instance": res.get("service.instance.id"), + "name": s["name"], "kind": s.get("kind", ""), "span_id": _hex(s.get("spanId")), + "parent_id": _hex(s.get("parentSpanId")), "trace_id": _hex(s.get("traceId")), + "start": start / 1e9, "end": end / 1e9, "duration_ms": round((end - start) / 1e6, 3), + "status": (s.get("status") or {}).get("code", "STATUS_CODE_UNSET"), + "attrs": {a["key"]: _attr_value(a["value"]) for a in s.get("attributes", [])}}) + spans.sort(key=lambda x: x["start"]) + return spans + + +def wait_trace(trace_id, predicate=None, timeout=60, interval=2): + """Ждём, пока трасса появится в Tempo (и удовлетворит predicate(spans)).""" + deadline = time.monotonic() + timeout + spans = [] + while time.monotonic() < deadline: + tr = tempo_trace(trace_id) + spans = flatten_spans(tr) if tr else [] + if spans and (predicate is None or predicate(spans)): + return spans + time.sleep(interval) + return spans + + +def tree(spans): + """Текстовое дерево трассы (для evidence): отступ = глубина, длительность, сервис.""" + by_parent = {} + ids = {s["span_id"] for s in spans} + for s in spans: + by_parent.setdefault(s["parent_id"] if s["parent_id"] in ids else "", []).append(s) + t0 = min((s["start"] for s in spans), default=0) + lines = [] + + def walk(pid, depth): + for s in sorted(by_parent.get(pid, []), key=lambda x: x["start"]): + lines.append("%s%-44s %-17s +%8.1fms %9.1fms %s%s" % ( + " " * depth, s["name"][:44], s["service"] or "", (s["start"] - t0) * 1000, s["duration_ms"], + s["kind"].replace("SPAN_KIND_", ""), " ERROR" if s["status"] == "STATUS_CODE_ERROR" else "")) + walk(s["span_id"], depth + 1) + walk("", 0) + return lines diff --git a/student-tests/outbox_gap.py b/student-tests/outbox_gap.py new file mode 100644 index 0000000..9570907 --- /dev/null +++ b/student-tests/outbox_gap.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Уровень 4: разрыв PostgreSQL <-> broker закрыт transactional outbox. Окна отказа: + + python3 student-tests/outbox_gap.py --output evidence/outbox-gap.json + + A. relay остановлен: COMMIT есть, publish нет -> outbox не опубликован, очередь пуста; + relay запущен -> задание доходит до ready без ручного создания нового задания; + B. RELAY_FAULT=crash-before-publish: relay выбрал строки outbox и умер до publish + (внедрённый crash между COMMIT и publish) -> рестарт контейнера -> ready; + C. RELAY_FAULT=crash-after-publish: publish + confirm прошли, published_at не записан -> + после рестарта сообщение опубликовано ещё раз -> worker гасит дубль, вариантов 3; + D. API crash-after-commit (FAULT_INJECTION=1): клиент не получил 202, но задание принято + и доходит до ready; + E. API crash-after-staging: оригинал в staging, задания нет (202 не было) -> pending-строку + и staging-объект убирает janitor после grace. +""" +import argparse +import json +import subprocess +import time + +from lab5lib import (Client, Failure, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, + queues, recreate, sql, upload, wait_status, wait_until) + + +def restart_count(service): + ids = compose("ps", "-q", "-a", service).stdout.split() + out = subprocess.run(["docker", "inspect", "-f", "{{.RestartCount}}"] + ids, capture_output=True, text=True) + return [int(x) for x in out.stdout.split()] + + +def wait_api_ready(): + """После внедрённого краха реплика перезапускается и снова проходит проверки LB (rise 2). + Не роняем следующую, пока обе не готовы: иначе на секунды не остаётся UP-реплик (503).""" + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "api", env={"FAULT_INJECTION": "1"}) + time.sleep(3) # rise 2 x inter 1s у HAProxy + + +def find_by_filename(name): + rows = sql("SELECT id, status, staging_key IS NOT NULL FROM attachments WHERE filename = '%s'" % name) + return rows[0] if rows else None + + +def raw_upload(c, actor, chat, name, fault): + """Процесс API умирает посреди запроса: ответ формирует LB (без X-Instance-Id), + поэтому здесь urllib напрямую, а не проверяющий контракт Client.call.""" + import urllib.error + import urllib.request + req = urllib.request.Request( + c.base + "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], name), data=SAMPLE_PNG, method="POST", + headers={"Content-Type": "image/png", "X-Fault-Inject": fault, "Authorization": "Bearer " + actor["token"]}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return {"http": r.status, "x_instance_id": r.headers.get("X-Instance-Id")} + except urllib.error.HTTPError as e: + body = e.read(2000) + return {"http": e.code, "x_instance_id": e.headers.get("X-Instance-Id"), + "error_code": json.loads(body)["error"]["code"] if body.startswith(b"{") else None} + except OSError as exc: + return {"connection_error": type(exc).__name__} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + args = p.parse_args() + log = Log("outbox: COMMIT/publish gap and API crash windows") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + res = {} + try: + # A --------------------------------------------------------------------------- + compose("stop", "relay") + att = upload(c, alice, chat, SAMPLE_PNG, "gap-relay-stopped.png") + time.sleep(3) + info = ops("inspect", att["id"]) + q = queues()["attachments.process.v1"] + check(info["outbox"][0]["published_at"] is None and get_att(c, bob, att["id"])["status"] == "queued", + "A: must stay unpublished") + log.step("A: relay stopped, upload 202", attachment_id=att["id"], outbox=info["outbox"], queue=q) + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "relay") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 60) + info = ops("inspect", att["id"]) + check(ready["status"] == "ready" and len(info["jobs"]) == 1, "A: not ready / extra job") + res["A"] = {"attachment_id": att["id"], "events": events(info), "outbox": info["outbox"], + "jobs": len(info["jobs"]), "invariants": invariants(info)} + log.step("A: relay started -> ready, same single job", **res["A"]) + + # B --------------------------------------------------------------------------- + recreate("relay", {"RELAY_FAULT": "crash-before-publish"}) + rc0 = restart_count("relay") + att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-before-publish.png") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90) + info = ops("inspect", att["id"]) + logs = compose("logs", "--no-log-prefix", "relay").stdout + check(ready["status"] == "ready" and "FAULT INJECTION crash-before-publish" in logs, "B") + res["B"] = {"attachment_id": att["id"], "relay_restart_count": [rc0, restart_count("relay")], + "events": events(info), "outbox": info["outbox"], "invariants": invariants(info)} + log.step("B: relay crashed between COMMIT and publish -> restarted -> ready", **res["B"]) + + # C --------------------------------------------------------------------------- + recreate("relay", {"RELAY_FAULT": "crash-after-publish"}) + att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-after-publish.png") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90) + info = wait_until(lambda: (lambda i: i if "duplicate_ignored" in events(i) or + "busy_ignored" in events(i) else None)(ops("inspect", att["id"])), + 60, 1, "duplicate handled") + inv = invariants(info) + check(ready["status"] == "ready" and inv["variants_ok"] and inv["final_objects_ok"], "C") + res["C"] = {"attachment_id": att["id"], "relay_restart_count": restart_count("relay"), + "events": events(info), "outbox": info["outbox"], "invariants": inv} + log.step("C: published twice (crash before marking) -> duplicate is a no-op", **res["C"]) + recreate("relay") + + # D --------------------------------------------------------------------------- + recreate("api", {"FAULT_INJECTION": "1"}) + name = "gap-api-crash-after-commit-%d.png" % time.time() + outcome = raw_upload(c, alice, chat, name, "crash-after-commit") + wait_api_ready() + row = wait_until(lambda: find_by_filename(name), 20, what="attachment row") + ready = wait_status(c, bob, row[0], {"ready", "failed"}, 90) + info = ops("inspect", row[0]) + check(ready["status"] == "ready", "D") + res["D"] = {"attachment_id": row[0], "client_saw": outcome, "events": events(info), + "invariants": invariants(info)} + log.step("D: API died after COMMIT before 202 -> work still completed", **res["D"]) + + # E --------------------------------------------------------------------------- + name = "gap-api-crash-after-staging-%d.png" % time.time() + outcome = raw_upload(c, alice, chat, name, "crash-after-staging") + wait_api_ready() + row = wait_until(lambda: find_by_filename(name), 20, what="pending row") + jobs_n = sql("SELECT count(*) FROM jobs WHERE attachment_id = '%s'" % row[0])[0][0] + check(row[1] == "pending" and jobs_n == "0", "E: expected pending row without job") + c.call("GET", "/api/v1/attachments/" + row[0], 404, actor=bob) + log.step("E: API died after staging before COMMIT", attachment_id=row[0], client_saw=outcome, + status=row[1], jobs=int(jobs_n), api_get=404) + time.sleep(3) + dry = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once", "--dry-run", + "--grace-seconds", "2").stdout + real = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once", + "--grace-seconds", "2").stdout + dry, real = json.loads(dry.strip().splitlines()[-1]), json.loads(real.strip().splitlines()[-1]) + gone = find_by_filename(name) is None + check(row[0] in dry["abandoned_ids"] and gone, "E: janitor did not clean") + res["E"] = {"attachment_id": row[0], "janitor_dry_run": {k: dry[k] for k in ("abandoned_pending", "orphan_objects", + "deleted_objects", "deleted_rows")}, + "janitor": {k: real[k] for k in ("abandoned_pending", "orphan_objects", "deleted_objects", + "deleted_rows", "missing_objects")}, "row_gone": gone} + log.step("E: janitor removed pending row and staging object", **res["E"]) + finally: + recreate("relay") + recreate("api") + log.step("relay and api restored to .env configuration") + log.save(args.output, results=res) + print("PASS: outbox closes COMMIT/publish gap; API crash windows handled") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_all.sh b/student-tests/run_all.sh new file mode 100755 index 0000000..bd763d3 --- /dev/null +++ b/student-tests/run_all.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Полный прогон проверок lab7 на запущенном стенде с записью результатов в evidence/. +# ./student-tests/run_all.sh — обязательное + lab7 (≈ 10 мин) +# WITH_LAB6=1 ./student-tests/run_all.sh — + перенесённые проверки lab5/lab6 поверх TLS (≈ 25 мин) +# Требуется: стенд поднят (REPORT.md «Запуск»), OpenBao unsealed, $OPERATOR_DIR доступен +# (restart.sh и lab7_openbao.py делают unseal после рестарта OpenBao). +set -u +cd "$(dirname "$0")/.." +export CA_FILE=${CA_FILE:-$PWD/secrets/pki/public-ca.crt} +export BASE_URL=${BASE_URL:-https://localhost:8443} +mkdir -p evidence +fail=0 +run() { # run + name=$1; shift + echo "== $name ($(date -u +%FT%TZ))" + if "$@" > "evidence/$name.txt" 2>&1; then echo " PASS"; else echo " FAIL (evidence/$name.txt)"; fail=1; fi + tail -1 "evidence/$name.txt" +} +CHECKS="docker compose --profile tools run --rm -T checks" + +run make-check make check +run make-test make test CA_FILE="$CA_FILE" +run make-test-security make test-security CA_FILE="$CA_FILE" +run lab7-tls-matrix $CHECKS /app/student-tests/lab7_tls_matrix.py --output /app/evidence/lab7-tls-matrix.json +run lab7-api-security python3 student-tests/lab7_api_security.py --output evidence/lab7-api-security.json +run lab7-openbao python3 student-tests/lab7_openbao.py --output evidence/lab7-openbao.json +run lab7-hardening python3 student-tests/lab7_hardening.py --output evidence/lab7-hardening.json +run test-persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE="$CA_FILE" +run test-failover make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE="$CA_FILE" +./scripts/start-all.sh > /dev/null +run load python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +run scan-secrets python3 scripts/scan_secrets.py +if [ "${WITH_LAB6:-0}" = 1 ]; then + run check-rules ./scripts/check_rules.sh + run obs-correlation python3 student-tests/obs_correlation.py --output evidence/obs-correlation.json + run dashboard-check python3 student-tests/dashboard_check.py --output evidence/dashboard-check.json + run worker-outage python3 student-tests/worker_outage.py +fi +echo "== done: $([ $fail = 0 ] && echo ALL PASS || echo SOME FAILED)" +exit $fail diff --git a/student-tests/s3_tamper.py b/student-tests/s3_tamper.py new file mode 100644 index 0000000..6ccb794 --- /dev/null +++ b/student-tests/s3_tamper.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Учебная порча staging-объекта (контейнер checks, ключ api): подменяет байты на другие +байты той же длины. Нужна worker_faults.py (staging_corrupted) и api_negative_lab5.py (F09). + + docker compose --profile tools run --rm checks /app/student-tests/s3_tamper.py staging/ +""" +import io +import os +import re +import sys + +sys.path.insert(0, "/app/src") +os.environ.setdefault("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_api_access_key") +os.environ.setdefault("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_api_secret_key") +import objstore # noqa: E402 + +key = sys.argv[1] +# Только учебные ключи: staging и конечные объекты вложений (аудит F09: порча той же длины). +if not re.fullmatch(r"staging/[0-9a-f]{32}|attachments/[0-9a-f-]{36}/v1/[a-z]+(\.[a-z]+)?", key): + sys.exit("refusing: not a staging/attachment key") +store = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments")) +import time # noqa: E402 +for attempt in range(1, 6): # запись SeaweedFS после перезапусков иногда «прогревается» секунды + try: + resp = store.open(key) + data = resp.read() + resp.close() + resp.release_conn() + bad = bytes(b ^ 0x5A for b in data) + store.put(key, io.BytesIO(bad), len(bad), "application/octet-stream") + break + except objstore.StorageUnavailable: + if attempt == 5: + raise + time.sleep(2) +print("tampered", key, len(bad)) diff --git a/student-tests/telemetry_outage.py b/student-tests/telemetry_outage.py new file mode 100644 index 0000000..2a91afe --- /dev/null +++ b/student-tests/telemetry_outage.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Lab6, уровень 4, сценарий 3: недоступность telemetry backend не ломает сообщения, RAM ограничена. + + python3 student-tests/telemetry_outage.py --output evidence/telemetry-outage.json + +Под постоянной нагрузкой traffic.py (4 потока, каждый 4-й цикл — upload): + A. 40 с базовой линии; + B. stop tempo + loki на --backend-down с: Collector копит батчи в ограниченной очереди + (queue_size 100), retry до 60 с, затем отбрасывает — потери в otelcol_exporter_send_failed_* / + enqueue_failed_*; память Collector ограничена memory_limiter + лимитом контейнера; + C. start tempo + loki, 40 с на восстановление; + D. stop otel-collector на --collector-down с: SDK приложений не может экспортировать; очереди + BatchSpanProcessor/BatchLogRecordProcessor (2048) переполняются — потери считает SDK + (otel_sdk_processor_*_processed{error_type="queue_full"}), RSS API не растёт; + E. start otel-collector, 40 с. +Для каждой фазы: ответы API (коды, p50/p95/p99), docker stats (RSS) API/worker/Collector, +счётчики Collector и SDK. Ожидаем: 5xx = 0, RAM ограничена, потери посчитаны. +""" +import argparse +import re +import subprocess +import sys +import time + +from lab5lib import ROOT, Client, Failure, Log, check, compose +from traffic import Traffic +import obslib + + +def mem_mib(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}"], + cwd=ROOT, capture_output=True, text=True, timeout=60).stdout + res = {} + for line in out.splitlines(): + name, usage, cpu = line.split("|") + if not name.startswith("mtusi-lab6-"): + continue + m = re.match(r"([\d.]+)(KiB|MiB|GiB|B)", usage.strip()) + if not m: + continue + v = float(m.group(1)) * {"B": 1 / 2 ** 20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[m.group(2)] + short = name[len("mtusi-lab6-"):] + if any(short.startswith(x) for x in ("api-", "worker-", "otel-collector", "relay")): + res[short] = {"mem_mib": round(v, 1), "cpu": cpu} + return res + + +def counters(): + q = { + "collector_spans_send_failed": 'sum({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}) or vector(0)', + "collector_spans_enqueue_failed": 'sum({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}) or vector(0)', + "collector_logs_send_failed": 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}) or vector(0)', + "collector_logs_enqueue_failed": 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}) or vector(0)', + "collector_spans_sent": 'sum({__name__=~"otelcol_exporter_sent_spans(_total)?"}) or vector(0)', + "collector_queue_tempo": 'max(otelcol_exporter_queue_size{exporter=~".*tempo"}) or vector(0)', + "collector_queue_loki": 'max(otelcol_exporter_queue_size{exporter=~".*loki"}) or vector(0)', + "collector_rss_mib": 'max({__name__=~"otelcol_process_memory_rss(_bytes)?"}) / 1048576', + "collector_heap_mib": 'max(otelcol_process_runtime_heap_alloc_bytes) / 1048576', + "sdk_spans_dropped_queue_full": 'sum(otel_sdk_processor_span_processed_total{error_type="queue_full"}) or vector(0)', + "sdk_logs_dropped_queue_full": 'sum(otel_sdk_processor_log_processed_total{error_type="queue_full"}) or vector(0)', + "sdk_span_export_failed": 'sum(otel_sdk_exporter_span_exported_total{error_type!=""}) or vector(0)', + "sdk_log_export_failed": 'sum(otel_sdk_exporter_log_exported_total{error_type!=""}) or vector(0)', + "sdk_spans_processed_ok": 'sum(otel_sdk_processor_span_processed_total{error_type=""}) or vector(0)', + } + out = {} + for k, expr in q.items(): + try: + out[k] = round(obslib.prom_value(expr, default=0.0), 1) + except Exception as exc: + out[k] = "unavailable (%s)" % type(exc).__name__ + return out + + +def alert_states(): + try: + return sorted({"%s:%s%s" % (a["labels"]["alertname"], a["state"], + (":" + a["labels"]["job"]) if "job" in a["labels"] else "") + for a in obslib.prom_alerts() if "Telemetry" in a["labels"]["alertname"]}) + except Exception as exc: + return ["prometheus unavailable (%s)" % type(exc).__name__] + + +def watch(log, tr, phase, seconds, t_wall0, step=10): + rows = [] + t_end = time.time() + seconds + while time.time() < t_end: + time.sleep(min(step, max(0.1, t_end - time.time()))) + rows.append({"t": round(time.time() - t_wall0), "phase": phase, "mem": mem_mib(), "alerts": alert_states()}) + return rows + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + p.add_argument("--backend-down", type=int, default=120) + p.add_argument("--collector-down", type=int, default=90) + args = p.parse_args() + log = Log("telemetry backend / collector outage under load") + c = Client(args.base_url, timeout=30) + alice, _bob, _eve, chat = c.bootstrap() + t_wall0 = time.time() + tr = Traffic(c, alice, chat, threads=4, think=0.2, upload_every=4).start() + phases, samples = {}, [] + + def phase(name, seconds, action=None): + t0 = time.time() - t_wall0 + if action: + action() + before = counters() + samples.extend(watch(log, tr, name, seconds, t_wall0)) + t1 = time.time() - t_wall0 + stats = tr.window(t0, t1) + phases[name] = {"from_s": round(t0), "to_s": round(t1), "api": stats, "counters_at_start": before, + "mem_last": samples[-1]["mem"], "alerts_last": samples[-1]["alerts"]} + log.step("phase %s done" % name, api=stats, mem=samples[-1]["mem"], alerts=samples[-1]["alerts"]) + + def increase(expr, seconds): + # increase() корректно учитывает сброс счётчика при рестарте процесса (в отличие от разности). + return round(obslib.prom_value("sum(increase(%s[%ds])) or vector(0)" % (expr, seconds), default=0.0)) + + phase("A_baseline", 40) + phase("B_tempo_loki_down", args.backend_down, lambda: compose("stop", "tempo", "loki")) + phase("C_backends_restored", 40, lambda: compose("up", "-d", "--no-deps", "--no-recreate", "tempo", "loki")) + win_bc = int(time.time() - t_wall0 - phases["B_tempo_loki_down"]["from_s"]) + peak_queue = {r["metric"].get("exporter"): max(float(v[1]) for v in r["values"]) for r in obslib.prom_range( + "max by (exporter) (otelcol_exporter_queue_size)", time.time() - win_bc, time.time(), 5)} + loss_b = { + "spans_send_failed": increase('{__name__=~"otelcol_exporter_send_failed_spans(_total)?"}', win_bc), + "spans_queue_full": increase('{__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}', win_bc), + "logs_send_failed": increase('{__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}', win_bc), + "logs_queue_full": increase('{__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}', win_bc), + "spans_accepted": increase('{__name__=~"otelcol_receiver_accepted_spans(_total)?"}', win_bc), + "logs_accepted": increase('{__name__=~"otelcol_receiver_accepted_log_records(_total)?"}', win_bc), + "peak_exporter_queue": peak_queue, + "queue_capacity": obslib.prom_value("max(otelcol_exporter_queue_capacity)"), + "peak_heap_mib": round(obslib.prom_value("max_over_time(otelcol_process_runtime_heap_alloc_bytes[%ds]) / 1048576" % win_bc), 1), + } + log.step("collector-side loss while Tempo+Loki were down (increase over B+C)", **loss_b) + d_start = counters() # последние значения SDK-счётчиков до остановки Collector + phase("D_collector_down", args.collector_down, lambda: compose("stop", "otel-collector")) + phase("E_collector_restored", 45, lambda: compose("up", "-d", "--no-deps", "--no-recreate", "otel-collector")) + tr.stop() + time.sleep(15) # SDK досылает накопленные cumulative-счётчики после восстановления + # Пока Collector лежит, SDK не может отправить и сами метрики: счётчик ошибок экспорта + # (новый ряд error_type=ConnectionError) появляется в Prometheus одной точкой уже после + # восстановления, и increase() по окну его не видит (нет точки «до»). Процессы приложения + # в фазах D-E не перезапускаются, счётчики монотонны -> потери = значение после - значение до. + end_c = counters() + loss_d = { + "sdk_spans_export_failed": round(end_c["sdk_span_export_failed"] - d_start["sdk_span_export_failed"]), + "sdk_logs_export_failed": round(end_c["sdk_log_export_failed"] - d_start["sdk_log_export_failed"]), + "sdk_spans_queue_full": round(end_c["sdk_spans_dropped_queue_full"] - d_start["sdk_spans_dropped_queue_full"]), + "sdk_logs_queue_full": round(end_c["sdk_logs_dropped_queue_full"] - d_start["sdk_logs_dropped_queue_full"]), + "sdk_spans_processed_ok": round(end_c["sdk_spans_processed_ok"] - d_start["sdk_spans_processed_ok"]), + "sdk_export_error_types": sorted({r["metric"].get("error_type") for r in obslib.prom_query( + 'otel_sdk_exporter_span_exported_total{error_type!=""}')}), + } + log.step("SDK-side loss while the Collector was down (increase over D+E)", **loss_d) + final = counters() + log.step("telemetry counters after recovery", **final) + + # Оценка: бизнес-путь не пострадал, RAM ограничена, потери посчитаны. + total_5xx = sum(v["api"].get("codes", {}).get(k, 0) for v in phases.values() for k in v["api"].get("codes", {}) + if k.startswith("5") or k == "0") + max_mem = {} + for s in samples: + for k, v in s["mem"].items(): + key = re.sub(r"-\d+$", "", k) + max_mem[key] = max(max_mem.get(key, 0), v["mem_mib"]) + base_mem = {re.sub(r"-\d+$", "", k): v["mem_mib"] for k, v in phases["A_baseline"]["mem_last"].items()} + loss = {"collector_while_backends_down": loss_b, "sdk_while_collector_down": loss_d} + log.step("verdict", api_5xx_total=total_5xx, max_mem_mib=max_mem, baseline_mem_mib=base_mem, loss=loss) + check(total_5xx == 0, "API errors during telemetry outage: %d" % total_5xx) + check(max_mem.get("otel-collector", 0) < 256, "collector memory bounded by its limit") + check(loss_b["spans_send_failed"] + loss_b["spans_queue_full"] > 0, "collector loss must be counted") + check(loss_d["sdk_spans_export_failed"] + loss_d["sdk_spans_queue_full"] > 0, "SDK loss must be counted") + check(all(max_mem.get(k, 0) < base_mem.get(k, 0) + 60 for k in ("api", "worker") if k in base_mem), + "application RSS bounded during outage") + log.save(args.output, phases=phases, samples=samples, final_counters=final, loss=loss, max_mem_mib=max_mem, + baseline_mem_mib=base_mem, traffic_by_10s=tr.summary_by(10)) + print("PASS: telemetry outage: API 5xx=%d, max RSS %s, losses %s" % (total_5xx, max_mem, loss)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + compose("up", "-d", "--no-deps", "--no-recreate", "tempo", "loki", "otel-collector", ok=False) + sys.exit(1) diff --git a/student-tests/telemetry_persistence.py b/student-tests/telemetry_persistence.py new file mode 100644 index 0000000..a5df1fd --- /dev/null +++ b/student-tests/telemetry_persistence.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Lab6, уровень 3: история телеметрии переживает перезапуск хранилищ (persistent volumes). + + python3 student-tests/telemetry_persistence.py --output evidence/telemetry-persistence.json + +До рестарта: контролируемое сообщение -> его trace (Tempo), лог по request_id (Loki), значение +счётчика запросов API в момент t1 (Prometheus). Затем scripts/restart.sh перезапускает ВСЕ +хранилища (данных и телеметрии) и процессы с теми же volumes. После: тот же trace, тот же лог и то +же историческое значение в момент t1 читаются снова; dashboard и datasources на месте. +""" +import argparse +import subprocess +import sys +import time + +from lab5lib import ROOT, Client, Failure, Log, check, compose +import obslib + + +def started_at(): + out = {} + for svc in ("db", "prometheus", "loki", "tempo", "otel-collector", "grafana"): + cid = compose("ps", "-q", svc).stdout.split()[0] + out[svc] = subprocess.run(["docker", "inspect", "-f", "{{.State.StartedAt}}", cid], capture_output=True, + text=True, cwd=ROOT).stdout.strip() + return out + + +def wait_backends(timeout=180): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + st = {u: (obslib.grafana("/api/datasources/uid/%s/health" % u) or {}).get("status") + for u in ("prometheus", "loki", "tempo")} + if all(v == "OK" for v in st.values()): + return st + except Exception: + pass + time.sleep(3) + raise Failure("telemetry backends not healthy after restart") + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--output") + args = p.parse_args() + log = Log("telemetry history survives restart of all stores") + c = Client(timeout=30) + alice, _bob, _eve, chat = c.bootstrap() + r = c.message(alice, chat, "persistence probe") + rid, tid = r["headers"]["X-Request-Id"], r["headers"]["X-Trace-Id"] + t0 = time.time() + spans = obslib.wait_trace(tid, lambda s: any(x["name"] == "db add_message" for x in s)) + check(spans, "trace before restart") + time.sleep(12) # метрика экспортируется раз в 10 с + t1 = time.time() + q = 'sum(http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"})' + v1 = obslib.prom_value(q, at=t1) + lq = '{service_name="messenger-api"} | request_id="%s"' % rid + l1 = obslib.loki_query(lq, t0 - 60, t1 + 5) + check(l1 and v1, "log and metric before restart") + before = started_at() + log.step("before restart", trace_id=tid, spans=len(spans), request_id=rid, log_lines=len(l1), + metric_at_t1=v1, started_at=before) + proc = subprocess.run([str(ROOT / "scripts/restart.sh")], cwd=ROOT, capture_output=True, text=True, timeout=300) + check(proc.returncode == 0, "restart.sh failed") + health = wait_backends() + after = started_at() + check(all(after[k] != before[k] for k in before), "all stores really restarted") + log.step("restart.sh done", started_at=after, datasource_health=health) + spans2 = obslib.wait_trace(tid, timeout=60) + v2 = obslib.prom_value(q, at=t1) + l2 = obslib.loki_query(lq, t0 - 60, t1 + 5) + dash = obslib.grafana("/api/dashboards/uid/messenger-overview") + check(len(spans2) == len(spans), "same trace after restart (%d vs %d spans)" % (len(spans2), len(spans))) + check(v2 == v1, "same historical metric value at t1 (%s vs %s)" % (v2, v1)) + check(len(l2) == len(l1) and l2[0]["labels"].get("trace_id") == tid, "same log line after restart") + check(dash and dash["meta"].get("provisioned"), "dashboard provisioned after restart") + log.step("after restart: history preserved", spans=len(spans2), metric_at_t1=v2, log_lines=len(l2), + log_trace_id=l2[0]["labels"].get("trace_id"), dashboard_provisioned=dash["meta"].get("provisioned")) + c.call("GET", "/health/ready") + log.save(args.output) + print("PASS: trace, log and metric history survived restart of all stores") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Failure as exc: + print("FAIL:", exc) + sys.exit(1) diff --git a/student-tests/traffic.py b/student-tests/traffic.py new file mode 100644 index 0000000..78cf15b --- /dev/null +++ b/student-tests/traffic.py @@ -0,0 +1,140 @@ +"""Lab6: фоновая смешанная нагрузка для экспериментов с инцидентами (стандартная библиотека). + +Профиль (closed loop, `threads` потоков): каждый поток в цикле читает историю чата, отправляет +сообщение с Idempotency-Key и раз в `upload_every` итераций загружает маленький PNG (64x48, +уникальный цвет). Пауза `think` между итерациями. Ошибки не прерывают нагрузку: каждый ответ +учитывается по секундам (коды, latency), чтобы сопоставить с метриками/alerts. +Токены живут только в памяти, в результат не попадают. +""" +import base64 +import json +import threading +import time +import urllib.error +import urllib.request +import uuid + +from lab5lib import png + +BASE = "https://localhost:8443" + + +class Traffic: + def __init__(self, client, actor, chat, threads=4, think=0.2, upload_every=5, timeout=15, base=BASE): + self.c = client + self.actor = actor + self.chat = chat + self.threads = threads + self.think = think + self.upload_every = upload_every + self.timeout = timeout + self.base = client.base if hasattr(client, "base") else base + self.stop_ev = threading.Event() + self.lock = threading.Lock() + self.samples = [] # (t, kind, status, latency_s) + self.uploads = [] # (t, attachment_id) + self._threads = [] + self.started = None + + def _req(self, method, path, body=None, raw=None, headers=None): + h = {"Authorization": "Bearer " + self.actor["token"], "Accept": "application/json"} + if body is not None: + raw = json.dumps(body).encode() + h["Content-Type"] = "application/json" + h.update(headers or {}) + req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method) + t = time.monotonic() + try: + # Lab7: HTTPS — opener публичного клиента (проверка CA_FILE и hostname). + opener = getattr(self.c, "opener", None) + with (opener.open(req, timeout=self.timeout) if opener else urllib.request.urlopen(req, timeout=self.timeout)) as resp: + data = resp.read() + status = resp.status + except urllib.error.HTTPError as exc: + data, status = exc.read(), exc.code + except Exception: + data, status = b"", 0 # timeout / connection error + return status, time.monotonic() - t, data + + def _record(self, kind, status, latency): + with self.lock: + self.samples.append((time.monotonic() - self.started, kind, status, latency)) + + def _loop(self, idx): + i = 0 + cid = self.chat["id"] + while not self.stop_ev.is_set(): + i += 1 + st, lat, _ = self._req("GET", "/api/v1/chats/%s/messages?limit=20" % cid) + self._record("list", st, lat) + st, lat, _ = self._req("POST", "/api/v1/chats/%s/messages" % cid, body={"text": "load %d-%d" % (idx, i)}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + self._record("send", st, lat) + if self.upload_every and i % self.upload_every == 0: + img = png(64, 48, color=((idx * 40 + i) % 256, (i * 7) % 256, 90)) + st, lat, data = self._req("POST", "/api/v1/chats/%s/attachments?filename=load.png" % cid, raw=img, + headers={"Content-Type": "image/png"}) + self._record("upload", st, lat) + if st == 202: + try: + with self.lock: + self.uploads.append((time.monotonic() - self.started, json.loads(data)["id"])) + except Exception: + pass + self.stop_ev.wait(self.think) + + def start(self): + self.started = time.monotonic() + for i in range(self.threads): + t = threading.Thread(target=self._loop, args=(i,), daemon=True) + t.start() + self._threads.append(t) + return self + + def stop(self): + self.stop_ev.set() + for t in self._threads: + t.join(timeout=self.timeout + 5) + + def window(self, t_from, t_to): + with self.lock: + rows = [s for s in self.samples if t_from <= s[0] < t_to] + return summarize(rows) + + def summary_by(self, step=10): + with self.lock: + rows = list(self.samples) + if not rows: + return [] + end = max(r[0] for r in rows) + out = [] + t = 0.0 + while t <= end: + part = [r for r in rows if t <= r[0] < t + step] + s = summarize(part) + s["t"] = round(t, 1) + out.append(s) + t += step + return out + + +def summarize(rows): + if not rows: + return {"requests": 0} + lat = sorted(r[3] for r in rows if 200 <= r[2] < 300) + codes = {} + for r in rows: + codes[str(r[2])] = codes.get(str(r[2]), 0) + 1 + + def q(p): + if not lat: + return None + return round(lat[min(len(lat) - 1, int(p * len(lat)))] * 1000, 1) + errors_5xx = sum(v for k, v in codes.items() if k.startswith("5") or k == "0") + span = (max(r[0] for r in rows) - min(r[0] for r in rows)) or 1 + return {"requests": len(rows), "ok": len(lat), "codes": codes, "error_ratio_5xx": round(errors_5xx / len(rows), 4), + "ok_rps": round(len(lat) / span, 1), "p50_ms": q(0.50), "p95_ms": q(0.95), "p99_ms": q(0.99)} + + +def basic(username, password): + return "Basic " + base64.b64encode(("%s:%s" % (username, password)).encode()).decode() diff --git a/student-tests/transform_limits.py b/student-tests/transform_limits.py new file mode 100644 index 0000000..4c59f1d --- /dev/null +++ b/student-tests/transform_limits.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Лимиты worker на «плохие» входы: пиксели, память, CPU-время, повреждённые данные. +Запускает src/transform.py так же, как worker (подпроцесс), В КОНТЕЙНЕРЕ checks: + + docker compose --profile tools run --rm checks /app/student-tests/transform_limits.py \ + --output /app/evidence/transform-limits.json + +API отсекает > 10 MiB и > 40 Мпикс ещё до очереди (lab4); здесь проверяется вторая линия +обороны — сам worker, если в staging всё же оказался такой файл. +""" +import argparse +import io +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from PIL import Image + +TRANSFORM = "/app/src/transform.py" +EXPECT = {0: "ok", 3: "invalid_image", 4: "image_too_large", 5: "resource_limit"} + + +def run(data, ctype, env=None, max_pixels=40000000, timeout=60): + """Как worker: подпроцесс; timeout — wall-clock (у worker JOB_TIMEOUT_SECONDS).""" + with tempfile.TemporaryDirectory() as d: + src = os.path.join(d, "in") + Path(src).write_bytes(data) + out = os.path.join(d, "out") + os.mkdir(out) + e = dict(os.environ, **(env or {})) + t = time.monotonic() + try: + p = subprocess.run([sys.executable, TRANSFORM, src, out, ctype, str(max_pixels)], capture_output=True, + env=e, timeout=timeout) + rc = p.returncode + except subprocess.TimeoutExpired: + rc = "timeout" + dur = round(time.monotonic() - t, 2) + if rc == "timeout": + return {"exit": rc, "class": "processing_timeout", "seconds": round(time.monotonic() - t, 2)} + cls = EXPECT.get(rc, "killed_by_signal_%d" % -rc if isinstance(rc, int) and rc < 0 else str(rc)) + return {"exit": rc, "class": cls, "seconds": dur} + + +def png(size, mode="RGB", color=(10, 20, 30)): + buf = io.BytesIO() + Image.new(mode, size, color).save(buf, "PNG") + return buf.getvalue() + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--output") + args = p.parse_args() + ok = png((640, 480)) + big = png((6000, 6000)) # 36 Мпикс: допустимо, ~108 MiB в RGB + # Несжимаемый RGBA-шум ~40 Мпикс: декодирование, alpha-композиция и resize — секунды CPU. + buf = io.BytesIO() + n = Image.effect_noise((6300, 6300), 80) + Image.merge("RGBA", [n, n.transpose(Image.Transpose.ROTATE_90), n, n]).save(buf, "PNG", compress_level=1) + heavy = buf.getvalue() + # Аудит F02: progressive 4:4:4 ~40 Мпикс — допустимый вход, ~250 MiB на декодирование. + g = Image.radial_gradient("L").resize((6324, 6324)) + buf = io.BytesIO() + Image.merge("RGB", (g, g.rotate(90), g)).save(buf, "JPEG", quality=85, progressive=True, subsampling=0) + prog = buf.getvalue() + cases = [ + ("valid 640x480", ok, "image/png", {}, 40000000, "ok"), + ("40 Mpx progressive 4:4:4 JPEG, default limits", prog, "image/jpeg", {}, 40000000, "ok"), + ("49 Mpx PNG (> MAX_IMAGE_PIXELS)", png((7000, 7000)), "image/png", {}, 40000000, "image_too_large"), + ("36 Mpx PNG, normal limits", big, "image/png", {}, 40000000, "ok"), + ("36 Mpx PNG, memory limit 160 MiB", big, "image/png", {"TRANSFORM_MEMORY_LIMIT_MB": "160"}, 40000000, + "resource_limit"), + ("40 Mpx RGBA noise PNG, CPU limit 1 s", heavy, "image/png", {"TRANSFORM_CPU_SECONDS": "1"}, 40000000, + "killed_by_signal_24"), + ("40 Mpx RGBA noise PNG, wall timeout 0.5 s", heavy, "image/png", {}, 40000000, "processing_timeout"), + ("truncated PNG", ok[: len(ok) // 2], "image/png", {}, 40000000, "invalid_image"), + ("garbage declared as JPEG", b"\xff\xd8\xff" + os.urandom(5000), "image/jpeg", {}, 40000000, "invalid_image"), + ("PNG declared as JPEG", ok, "image/jpeg", {}, 40000000, "invalid_image"), + ] + results = [] + failed = 0 + for name, data, ctype, env, mp, want in cases: + r = run(data, ctype, env, mp, timeout=0.5 if want == "processing_timeout" else 60) + r.update(case=name, expected=want, input_bytes=len(data), env=env) + r["pass"] = r["class"] == want or (want.startswith("killed") and r["class"] in ("resource_limit", want)) + failed += not r["pass"] + results.append(r) + print("%s %-36s -> %s (%.2fs)" % ("PASS" if r["pass"] else "FAIL", name, r["class"], r["seconds"]), flush=True) + if args.output: + Path(args.output).write_text(json.dumps({"cases": results}, ensure_ascii=False, indent=1) + "\n") + print("PASS" if not failed else "FAIL", "%d/%d" % (len(cases) - failed, len(cases))) + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/student-tests/worker_faults.py b/student-tests/worker_faults.py new file mode 100644 index 0000000..43e8b03 --- /dev/null +++ b/student-tests/worker_faults.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Уровень 4: отказы worker, retry/backoff, failed + replay, DLQ. + + python3 student-tests/worker_faults.py --output evidence/worker-faults.json + +На время теста worker пересоздаются с WORKER_FAULT_INJECTION=1 и JOB_RETRY_BASE_SECONDS=1 +(короче ожидание backoff); в конце возвращаются к значениям из .env. +Точка отказа задаётся именем файла fault-.png (src/worker.py). + + crash-after-claim процесс умер в processing до любых эффектов + crash-after-s3-put все объекты записаны в S3, COMMIT ready не сделан, ack не отправлен + crash-after-commit ready зафиксирован, ack не отправлен (и staging не удалён) + transient-2 временная ошибка в попытках 1–2: backoff с jitter, ready с 3-й + bug непредусмотренное исключение в 1-й попытке -> processing_failed -> retry -> ready + transient-always временная ошибка до replay: 5 попыток -> failed(retry_exhausted) + -> content 409, прикрепление 409 -> ops replay -> ready; повторный replay отклонён + staging tamper staging подменён до обработки -> failed(staging_corrupted) без повторов + poison message невалидное сообщение -> reject -> DLQ -> ops dlq --purge-invalid +Инварианты после каждого: 3 варианта, 4 конечных объекта, staging убран (с janitor). +""" +import argparse +import json +import subprocess +import time + +from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, png, queues, + recreate, upload, wait_status, wait_until) + +FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"} + + +def inspect_until(att_id, pred, timeout=60, what="state"): + return wait_until(lambda: (lambda i: i if pred(i) else None)(ops("inspect", att_id)), timeout, 1, what) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + args = p.parse_args() + log = Log("worker faults, retries, failed registry, replay, DLQ") + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + recreate("worker", FAULT_ENV) + log.step("workers recreated with fault injection", env=FAULT_ENV) + report = {} + try: + # --- крахи процесса в разных окнах ------------------------------------------------- + for point in ("crash-after-claim", "crash-after-s3-put", "crash-after-commit"): + att = upload(c, alice, chat, SAMPLE_PNG, "fault-%s.png" % point) + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90) + check(ready["status"] == "ready", point + ": not ready") + if point == "crash-after-commit": + # ready уже был зафиксирован; redelivery после рестарта -> no-op ack. + inspect_until(att["id"], lambda i: "duplicate_ignored" in events(i), 60, "redelivery no-op") + compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once") + info = ops("inspect", att["id"]) + inv = invariants(info) + check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], point + ": invariants") + report[point] = {"attachment_id": att["id"], "events": events(info), "invariants": inv} + log.step(point, events=events(info), invariants=inv) + ids = compose("ps", "-q", "worker").stdout.split() + rc = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.Status}}"] + ids, + capture_output=True, text=True).stdout.split("\n") + log.step("worker containers after crashes (restart: unless-stopped)", containers=[x for x in rc if x]) + + # --- временные ошибки: backoff + jitter --------------------------------------------- + att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-2.png") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90) + check(ready["status"] == "ready", "transient-2 not ready") + info = ops("inspect", att["id"]) + delays = [e["detail"]["delay_seconds"] for e in info["events"] if e["event"] == "retry_scheduled"] + claims = [e["at"] for e in info["events"] if e["event"] == "claimed"] + bounds = [[0.5 * min(8, 2 ** (n - 1)), min(8, 2 ** (n - 1))] for n in (1, 2)] + check(len(delays) == 2 and all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "backoff bounds") + inv = invariants(info) + check(inv["attempt"] == 3 and inv["variants_ok"], "transient-2 attempts") + report["transient-2"] = {"attachment_id": att["id"], "delays": delays, "bounds": bounds, "claims_at": claims, + "invariants": inv} + log.step("transient-2: retried with backoff", delays=delays, bounds=bounds, invariants=inv) + + # --- непредусмотренное исключение: штатный retry, а не «висящий» processing ---------- + att = upload(c, alice, chat, SAMPLE_PNG, "fault-bug.png") + t0 = time.monotonic() + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60) + took = round(time.monotonic() - t0, 2) + info = ops("inspect", att["id"]) + codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"] + check(ready["status"] == "ready" and codes == ["processing_failed"] and took < 15, "bug: retry path") + report["bug"] = {"attachment_id": att["id"], "retry_error_codes": codes, "seconds_to_ready": took, + "events": events(info), "invariants": invariants(info)} + log.step("unexpected exception -> processing_failed -> retry -> ready", **report["bug"]) + + # --- исчерпание попыток -> failed -> replay ---------------------------------------- + att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png") + failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "expected retry_exhausted") + r409 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + m409 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "f-" + att["id"]}) + registry = ops("failed") + in_registry = [r for r in registry["failed"] if r["attachment_id"] == att["id"]] + check(in_registry and in_registry[0]["replayable"], "failed registry") + info = ops("inspect", att["id"]) + log.step("transient-always: failed after max attempts", error_code=failed["error_code"], + attempts=info["jobs"][0]["attempt"], content=r409["status"], attach=m409["status"], + registry_entry=in_registry[0]) + dry = ops("replay", att["id"], "--dry-run") + rep = ops("replay", att["id"], "--reason", "test: S3 restored") + check(rep["replay"][0]["result"] == "replayed", "replay") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60) + check(ready["status"] == "ready", "not ready after replay") + again = ops("replay", att["id"]) + check(again["replay"][0]["result"] == "refused_status_ready", "replay of ready must be refused") + info = ops("inspect", att["id"]) + inv = invariants(info) + check(inv["variants_ok"] and inv["final_objects_ok"], "replay invariants") + report["transient-always+replay"] = {"attachment_id": att["id"], "dry_run": dry["replay"][0]["result"], + "replay": rep["replay"][0]["result"], + "second_replay": again["replay"][0]["result"], + "replay_count": info["jobs"][0]["replay_count"], + "events": events(info), "invariants": inv} + log.step("replay -> ready; replay of ready refused", **report["transient-always+replay"]) + + # --- подмена staging: постоянная ошибка без повторов -------------------------------- + compose("stop", "worker") + att = upload(c, alice, chat, png(50, 40), "tamper.png") + info = ops("inspect", att["id"]) + key = info["attachment"]["staging_key"] + compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key) + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", env=FAULT_ENV) + failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60) + info = ops("inspect", att["id"]) + check(failed["error_code"] == "staging_corrupted" and info["jobs"][0]["attempt"] == 1, "tamper -> permanent") + report["staging-tamper"] = {"attachment_id": att["id"], "error_code": failed["error_code"], + "attempts": info["jobs"][0]["attempt"], "events": events(info)} + log.step("tampered staging -> failed without retries", **report["staging-tamper"]) + + # --- poison message -> DLQ ---------------------------------------------------------- + before = queues()["attachments.dlq"]["messages"] + ops("publish-raw", json.dumps({"schema_version": 1, "hello": "not a job"})) + q = wait_until(lambda: (lambda x: x if x["messages"] > before else None)(queues()["attachments.dlq"]), + 30, what="message in DLQ") + listed = ops("dlq") + purged = ops("dlq", "--purge-invalid") + # Счётчики quorum-очереди в rabbitmqctl обновляются с задержкой в секунды. + after = wait_until(lambda: (lambda n: "0" if n == 0 else None)(queues()["attachments.dlq"]["messages"]), + 20, what="empty DLQ") and 0 + report["poison"] = {"dlq_before": before, "dlq_after_publish": q["messages"], "listed": listed["dlq_messages"], + "purged": [m["action"] for m in purged["dlq_messages"]], "dlq_after_purge": after} + log.step("poison message dead-lettered and purged", **report["poison"]) + check(after == 0, "DLQ not empty after purge") + + # --- жёсткий срок задания (аудит F13): JOB_TIMEOUT 5 с, «долгая» работа 8 с ---------- + recreate("worker", dict(FAULT_ENV, JOB_TIMEOUT_SECONDS="5", FAULT_SLOW_SECONDS="8")) + att = upload(c, alice, chat, SAMPLE_PNG, "fault-slow.png") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90) + info = ops("inspect", att["id"]) + codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"] + check(ready["status"] == "ready" and codes == ["processing_timeout"], "deadline: %s" % codes) + report["deadline"] = {"attachment_id": att["id"], "retry_error_codes": codes, "events": events(info), + "invariants": invariants(info)} + log.step("job deadline exceeded -> processing_timeout -> retry -> ready", **report["deadline"]) + finally: + recreate("worker") + log.step("workers restored to .env configuration") + log.save(args.output, results=report) + print("PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ") + + +if __name__ == "__main__": + main() diff --git a/student-tests/worker_outage.py b/student-tests/worker_outage.py new file mode 100644 index 0000000..334675d --- /dev/null +++ b/student-tests/worker_outage.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Уровень 3, сценарий защиты: worker остановлен -> upload 202 -> сообщение с вложением -> +рестарт API и брокера (volumes сохраняются) -> worker запущен -> ready -> повторная доставка. + + python3 student-tests/worker_outage.py --output evidence/worker-outage.json + +Проверяется: + * 202 queued при остановленных worker; GET queued; content и thumbnail -> 409; + * сообщение с queued-вложением и обычное сообщение -> 201 (остальной API работает); + * задание лежит в durable quorum-очереди (messages_ready=1) и переживает рестарт брокера; + * после рестарта API и брокера и запуска worker вложение ready, thumbnail 256x160, + оригинал побайтно совпадает; + * повторная доставка (тот же job_id ещё раз в очередь) -> no-op, вариантов по-прежнему 3. +""" +import argparse +import hashlib +import json + +from lab5lib import (SAMPLE_PNG, Client, Log, check, compose, events, get_att, invariants, ops, queues, + upload, wait_status, wait_until) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=None) + p.add_argument("--output") + args = p.parse_args() + log = Log("worker outage -> restart api+broker -> worker start -> ready -> redelivery") + c = Client(args.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + + compose("stop", "worker") + log.step("worker stopped", running=compose("ps", "-q", "--status", "running", "worker").stdout.split()) + before = queues()["attachments.process.v1"] + + att = upload(c, alice, chat, SAMPLE_PNG, "outage.png") + check(att["status"] == "queued", "expected queued") + log.step("upload while worker is down", http=202, attachment_id=att["id"], status=att["status"]) + meta = get_att(c, bob, att["id"]) + r1 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + r2 = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob) + log.step("metadata and content while queued", status=meta["status"], content=r1["status"], + thumbnail=r2["status"], error_code=r1["json"]["error"]["code"]) + m1 = c.message(alice, chat, "", attachments=[att["id"]])["json"] + m2 = c.message(bob, chat, "другие сообщения работают")["json"] + log.step("messages while worker is down", with_queued_attachment=m1["attachment_ids"] == [att["id"]], + plain_message_id=m2["id"]) + + q = wait_until(lambda: (lambda x: x if x["messages_ready"] >= before["messages_ready"] + 1 else None)( + queues()["attachments.process.v1"]), 15, what="job message in queue") + info = ops("inspect", att["id"]) + log.step("job is durable in broker", queue=q, outbox_published=info["outbox"][0]["published_at"] is not None, + job_id=info["jobs"][0]["id"], job_status=info["jobs"][0]["status"]) + job_id = info["jobs"][0]["id"] + + compose("restart", "api") + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "api") + log.step("api restarted", status=get_att(c, alice, att["id"])["status"]) + compose("restart", "broker") + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "broker") + q2 = queues()["attachments.process.v1"] + check(q2["messages_ready"] >= 1, "job message lost after broker restart") + log.step("broker restarted with volume", queue=q2, status=get_att(c, alice, att["id"])["status"]) + + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker") + log.step("worker started") + ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60) + check(ready["status"] == "ready" and ready["error_code"] is None, "attachment not ready") + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"] + tb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"] + check(content == SAMPLE_PNG, "original changed") + check(hashlib.sha256(tb).hexdigest() == thumb["sha256"], "thumbnail checksum") + log.step("ready after worker start", variants=[(v["name"], v["width"], v["height"]) for v in ready["variants"]], + original_sha256_ok=True, thumbnail_bytes=len(tb)) + + # Повторная доставка: то же сообщение (тот же job_id) ещё раз в очередь. + payload = {"schema_version": 1, "job_id": job_id, "type": "attachment.process.v1", "attachment_id": att["id"], + "pipeline_version": 1, "attempt": 1, "created_at": "2026-11-13T00:00:00Z"} + ops("publish-raw", json.dumps(payload)) + info = wait_until(lambda: (lambda i: i if "duplicate_ignored" in events(i) else None)(ops("inspect", att["id"])), + 30, interval=1, what="duplicate_ignored event") + inv = invariants(info) + check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], "invariants violated") + log.step("redelivery of the same job is a no-op", events=events(info), invariants=inv) + after = get_att(c, bob, att["id"]) + check(after == ready, "metadata changed after redelivery") + log.save(args.output, attachment_id=att["id"], job_id=job_id, events=info["events"]) + print("PASS: worker outage, api+broker restart, ready, duplicate delivery no-op") + + +if __name__ == "__main__": + main()