From 861e7722b5ad21c939974aebdfb4079736d4804f Mon Sep 17 00:00:00 2001 From: alenchha Date: Mon, 16 Nov 2026 10:49:13 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=207=20=E2=80=94=20=D0=91=D0=92?= =?UTF-8?q?=D0=A22301=20=E2=80=94=20=D0=9A=D1=83=D1=80=D0=BB=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=20=E2=80=94=20=D0=BD=D0=B0=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия: - внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial; - TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL), Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA; - OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь; - роли БД, S3-политики и права брокера по принципу минимальных прав; - Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS; - hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия; - отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao. --- .env.example | 70 +- .github/workflows/stand.yml | 48 + Dockerfile | 35 + REPORT.md | 207 +- app/__init__.py | 0 app/broker.py | 81 + app/config.py | 134 ++ app/cursor.py | 43 + app/db.py | 145 ++ app/errors.py | 37 + app/faults.py | 38 + app/imagecheck.py | 213 ++ app/imageproc.py | 115 ++ app/jobs.py | 340 +++ app/main.py | 801 ++++++++ app/openbao.py | 161 ++ app/outbox.py | 100 + app/passwords.py | 29 + app/reconcile.py | 155 ++ app/repo.py | 342 ++++ app/s3.py | 132 ++ app/serve.py | 95 + app/sessions.py | 211 ++ app/telemetry.py | 393 ++++ app/worker.py | 373 ++++ compose.yaml | 604 ++++++ contracts/trust-matrix.md | 85 +- docs/dashboard.md | 244 +++ docs/runbooks.md | 90 + evidence/auth_acl.txt | 11 + evidence/bao_sealed.txt | 8 + evidence/cert_rotation.txt | 11 + evidence/dashboard_check.txt | 92 + evidence/failover.txt | 13 + evidence/hardening.json | 342 ++++ evidence/hardening.txt | 18 + evidence/image_scan.txt | 16 + evidence/lab5_failure_windows.txt | 80 + evidence/lab6_worker_down.txt | 14 + evidence/login_budget.json | 87 + evidence/observability.txt | 27 + evidence/persistence.txt | 36 + evidence/secret_rotation.txt | 8 + evidence/smoke.txt | 2 + evidence/smoke_check_validate_security.txt | 9 + evidence/tls_audit.json | 1082 ++++++++++ evidence/tls_audit.txt | 124 ++ evidence/tls_audit_final.txt | 139 ++ evidence/tls_audit_side_channels.txt | 15 + evidence/variants_limits.txt | 20 + infra/grafana/dashboards/messenger.json | 1823 +++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 9 + .../provisioning/datasources/datasources.yaml | 63 + infra/loki/loki.yaml | 48 + infra/minio/init.sh | 19 + infra/minio/policy-api.json | 23 + infra/minio/policy-worker.json | 17 + infra/nginx-health.sh | 65 + infra/nginx.conf | 95 + infra/openbao/README.md | 43 +- infra/openbao/config.hcl | 21 + infra/openbao/policies/messenger-api.hcl | 7 + infra/openbao/policies/messenger-worker.hcl | 7 + infra/otel/collector.yaml | 111 + infra/otel/profile-default.yaml | 5 + infra/otel/profile-tail.yaml | 31 + infra/postgres/pg_hba.conf | 8 + infra/prometheus/health-client.yml | 5 + infra/prometheus/prometheus.yml | 25 + infra/prometheus/rules/messenger.yml | 111 + infra/prometheus/web.yml | 9 + infra/rabbitmq/definitions.json | 25 + infra/rabbitmq/enabled_plugins | 1 + infra/rabbitmq/rabbitmq.conf | 25 + infra/tempo/tempo.yaml | 40 + infra/valkey/valkey-start.sh | 24 + migrations/0001_init.sql | 46 + migrations/0002_idempotency.sql | 12 + migrations/0003_attachments.sql | 28 + migrations/0004_allow_empty_text.sql | 2 + migrations/0005_attachment_upload_state.sql | 5 + migrations/0006_async_processing.sql | 75 + migrations/0007_service_roles.sql | 29 + requirements.txt | 60 + scripts/bao.sh | 126 ++ scripts/gen_dashboard.py | 244 +++ scripts/init-secrets.sh | 34 + scripts/jobs.py | 88 + scripts/migrate.py | 78 + scripts/pki.sh | 190 ++ scripts/reconcile.py | 32 + scripts/restart.sh | 15 + scripts/stop-one.sh | 5 + security/THREAT_MODEL.md | 113 +- student-tests/check_dashboard.py | 79 + student-tests/check_hardening.py | 79 + student-tests/login_budget.py | 157 ++ student-tests/mtls_probe.py | 15 + student-tests/obs.py | 146 ++ student-tests/openloop.py | 211 ++ student-tests/operator_checks.sh | 42 + student-tests/queue_load.py | 250 +++ student-tests/stand.py | 138 ++ student-tests/test_auth_acl.py | 135 ++ student-tests/test_bao_sealed.py | 71 + student-tests/test_cert_rotation.py | 88 + student-tests/test_exporter_queue_full.py | 58 + student-tests/test_failure_windows.py | 194 ++ student-tests/test_incidents.py | 269 +++ student-tests/test_observability.py | 159 ++ student-tests/test_secret_rotation.py | 88 + student-tests/test_worker_down.py | 84 + student-tests/test_worker_saturation.py | 75 + student-tests/tls_audit.py | 461 +++++ student-tests/track1.py | 343 ++++ student-tests/variants_check.py | 126 ++ 116 files changed, 14522 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/stand.yml create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/broker.py create mode 100644 app/config.py create mode 100644 app/cursor.py create mode 100644 app/db.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/imagecheck.py create mode 100644 app/imageproc.py create mode 100644 app/jobs.py create mode 100644 app/main.py create mode 100644 app/openbao.py create mode 100644 app/outbox.py create mode 100644 app/passwords.py create mode 100644 app/reconcile.py create mode 100644 app/repo.py create mode 100644 app/s3.py create mode 100644 app/serve.py create mode 100644 app/sessions.py create mode 100644 app/telemetry.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 docs/dashboard.md create mode 100644 docs/runbooks.md create mode 100644 evidence/auth_acl.txt create mode 100644 evidence/bao_sealed.txt create mode 100644 evidence/cert_rotation.txt create mode 100644 evidence/dashboard_check.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/hardening.json create mode 100644 evidence/hardening.txt create mode 100644 evidence/image_scan.txt create mode 100644 evidence/lab5_failure_windows.txt create mode 100644 evidence/lab6_worker_down.txt create mode 100644 evidence/login_budget.json create mode 100644 evidence/observability.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/secret_rotation.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/smoke_check_validate_security.txt create mode 100644 evidence/tls_audit.json create mode 100644 evidence/tls_audit.txt create mode 100644 evidence/tls_audit_final.txt create mode 100644 evidence/tls_audit_side_channels.txt create mode 100644 evidence/variants_limits.txt create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/loki.yaml create mode 100755 infra/minio/init.sh create mode 100644 infra/minio/policy-api.json create mode 100644 infra/minio/policy-worker.json create mode 100644 infra/nginx-health.sh create mode 100644 infra/nginx.conf create mode 100644 infra/openbao/config.hcl create mode 100644 infra/openbao/policies/messenger-api.hcl create mode 100644 infra/openbao/policies/messenger-worker.hcl create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/profile-default.yaml create mode 100644 infra/otel/profile-tail.yaml create mode 100644 infra/postgres/pg_hba.conf create mode 100644 infra/prometheus/health-client.yml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules/messenger.yml create mode 100644 infra/prometheus/web.yml create mode 100644 infra/rabbitmq/definitions.json create mode 100644 infra/rabbitmq/enabled_plugins create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 infra/tempo/tempo.yaml create mode 100644 infra/valkey/valkey-start.sh create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_allow_empty_text.sql create mode 100644 migrations/0005_attachment_upload_state.sql create mode 100644 migrations/0006_async_processing.sql create mode 100644 migrations/0007_service_roles.sql create mode 100644 requirements.txt create mode 100755 scripts/bao.sh create mode 100644 scripts/gen_dashboard.py create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/jobs.py create mode 100644 scripts/migrate.py create mode 100755 scripts/pki.sh create mode 100644 scripts/reconcile.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/check_dashboard.py create mode 100644 student-tests/check_hardening.py create mode 100644 student-tests/login_budget.py create mode 100644 student-tests/mtls_probe.py create mode 100644 student-tests/obs.py create mode 100644 student-tests/openloop.py create mode 100755 student-tests/operator_checks.sh create mode 100644 student-tests/queue_load.py create mode 100755 student-tests/stand.py create mode 100644 student-tests/test_auth_acl.py create mode 100644 student-tests/test_bao_sealed.py create mode 100644 student-tests/test_cert_rotation.py create mode 100644 student-tests/test_exporter_queue_full.py create mode 100755 student-tests/test_failure_windows.py create mode 100644 student-tests/test_incidents.py create mode 100644 student-tests/test_observability.py create mode 100644 student-tests/test_secret_rotation.py create mode 100755 student-tests/test_worker_down.py create mode 100644 student-tests/test_worker_saturation.py create mode 100644 student-tests/tls_audit.py create mode 100644 student-tests/track1.py create mode 100644 student-tests/variants_check.py diff --git a/.env.example b/.env.example index 2e9f054..fa8d5d3 100644 --- a/.env.example +++ b/.env.example @@ -1,41 +1,67 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=https://s3:9000 +GRACEFUL_TIMEOUT=8 +MAX_INFLIGHT_REQUESTS=64 +MAX_BODY_BYTES=65536 +MAX_RSS_MB=0 +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT=3 +DB_CONNECT_TIMEOUT=3 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_OP_TIMEOUT=6 +REQUEST_TIMEOUT=30 +DB_READY_TIMEOUT=1.5 +READY_CHECK_SECONDS=1 +READY_FAILURES=2 +SESSION_CACHE_TTL_SECONDS=30 +SESSION_SWEEP_SECONDS=60 +CACHE_TIMEOUT=0.2 +SESSION_REVOKED_TTL_SECONDS=60 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +UPLOAD_CONCURRENCY=2 +RECONCILE_INTERVAL_SECONDS=300 +RECONCILE_GRACE_SECONDS=900 +# --- lab5: broker, jobs, worker WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=20 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=30 +JOB_SWEEP_SECONDS=5 +WORKER_CHILD_MEMORY_MB=1024 STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab -# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix. +# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing. +MAX_PENDING_JOBS=200 +JPEG_QUALITY=85 +WEBP_QUALITY=80 +# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else. +FAULT_INJECTION=1 +# Session store: PostgreSQL is the source of truth, Valkey (TLS + ACL user `api`) caches sessions and holds the +# shared login rate limit counters. +SESSION_STORE=valkey +# --- lab7: TLS and OpenBao. No passwords or tokens here: compose.yaml passes only file PATHS (Docker secrets). PUBLIC_BASE_URL=https://localhost:8443 -TLS_CA_FILE=/run/secrets/ca.crt -TLS_CERT_FILE=/run/secrets/service.crt -TLS_KEY_FILE=/run/secrets/service.key -BAO_ADDR=https://openbao:8200 -BAO_CACERT=/run/secrets/bao-ca.crt -# No root token here. Bootstrap operator -> short-lived service auth -> scoped token. +# Argon2id (RFC 9106 / OWASP minimum: m=19 MiB, t=2, p=1). Measured on the stand, see REPORT.md. ARGON2_MEMORY_KIB=19456 ARGON2_ITERATIONS=2 ARGON2_PARALLELISM=1 +# Concurrent password hashes per replica (login DoS budget) and login attempts per window, shared via Valkey. +PASSWORD_HASH_CONCURRENCY=4 +LOGIN_RATE_LIMIT_PER_USER=5 +LOGIN_RATE_LIMIT_PER_IP=120 +LOGIN_RATE_WINDOW_SECONDS=60 +# OpenBao: secrets are re-read on this period (rotation pick-up); startup waits this long for an unsealed OpenBao. +BAO_REFRESH_SECONDS=15 +BAO_STARTUP_TIMEOUT_SECONDS=120 + diff --git a/.github/workflows/stand.yml b/.github/workflows/stand.yml new file mode 100644 index 0000000..8977001 --- /dev/null +++ b/.github/workflows/stand.yml @@ -0,0 +1,48 @@ +name: Application stand (lab7) +# Student job: the whole operator bootstrap in a throwaway runner, then the public HTTPS checks and the TLS audit. +# The job itself plays the operator: PKI, bootstrap passwords and OpenBao shares are created inside the runner and +# destroyed with it. No repository secrets are used; pull_request_target is deliberately not a trigger. +on: [push, pull_request] +permissions: + contents: read +jobs: + stand: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Local PKI and bootstrap credentials (operator step) + run: | + sh scripts/pki.sh init + sh scripts/init-secrets.sh + - name: Validate Compose configuration + run: docker compose --env-file .env config --quiet + - name: OpenBao init, unseal and bootstrap (operator step) + run: | + docker compose --env-file .env up -d openbao + sh scripts/bao.sh init + sh scripts/bao.sh bootstrap + - name: Build and start the stand, wait for readiness + run: docker compose --env-file .env up -d --build --wait --wait-timeout 300 + - name: Contract structure and public HTTPS checks + run: | + make check + make test CA_FILE=$PWD/secrets/pki/ca.crt + make test-security CA_FILE=$PWD/secrets/pki/ca.crt + - name: TLS / mTLS / authorization audit of the trust matrix + run: > + docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro + -v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py + - name: Container hardening from docker inspect + run: python3 student-tests/check_hardening.py + - name: Service state and logs on failure (no secrets are logged) + if: failure() + run: | + docker compose ps -a + docker compose logs --tail 60 nginx api-1 api-2 worker openbao rabbitmq postgres + - name: Remove the stand + if: always() + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..96f2c1a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.12.16-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-deps -r requirements.txt + +COPY app/ app/ +COPY migrations/ migrations/ +COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/ + +USER 10001:10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"] + +STOPSIGNAL SIGTERM + +CMD ["python", "-m", "app.serve"] diff --git a/REPORT.md b/REPORT.md index 9801db4..11e0bfd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,210 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Алёна Курлова, БВТ2301 +- Целевая оценка; выбранный трек 5: **4** (трек 5 не заявлен) +- Commit/PR: после публикации (ветка `solution/bvt2301-kurlova`) +- Источники, библиотеки, помощь ИИ: OpenBao 2.6.5 ([AppRole/cert auth](https://openbao.org/docs/auth/cert/), [policies](https://openbao.org/docs/concepts/policies/), [production hardening](https://openbao.org/docs/internals/security/)), [OWASP Password Storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [OWASP File Upload](https://cheatsheetseries.owasp.org/cheatsheets/File_Upload_Cheat_Sheet.html), [Docker security](https://docs.docker.com/engine/security/), RabbitMQ [TLS](https://www.rabbitmq.com/docs/ssl) и [access control](https://www.rabbitmq.com/docs/access-control), PostgreSQL [SSL](https://www.postgresql.org/docs/17/ssl-tcp.html), argon2-cffi, Trivy 0.58.1. Код lab6 перенесён, бизнес-API не менялся. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора. Все результаты ниже получены на локальном стенде и сохранены в `evidence/`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Desktop (Compose v2.20+), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7.2, Docker VM 10 CPU / 7.7 GiB. + +**Шаги оператора** (секреты и ключи создаются локально, в Git и образы не попадают; скрипты ничего секретного не печатают): + +```sh +sh scripts/pki.sh init # CA (Root + Ingest + Telemetry Clients) и leaf-сертификаты -> secrets/pki/ (0700) +sh scripts/init-secrets.sh # .env и bootstrap-пароли хранилищ -> secrets/bootstrap/ (0700) +docker compose --env-file .env up -d openbao +sh scripts/bao.sh init # 3 доли unseal (порог 2) + root token -> secrets/operator/ (0600); затем unseal +sh scripts/bao.sh bootstrap # KV v2 `messenger/`, политики, cert auth (роли api/worker), секреты сервисов +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +make check +make test CA_FILE=$PWD/secrets/pki/ca.crt +make test-security CA_FILE=$PWD/secrets/pki/ca.crt +``` + +**Bootstrap, повторяемый при каждом `up`:** +- `migrate` — SQL-миграции, роли `app_api`/`app_worker` и их пароли из `secrets/bootstrap`; подключение по TLS `verify-full`; +- `minio-init` — бакет, политики, пользователи `messenger-api` и `messenger-worker`; по HTTPS; +- RabbitMQ при каждом старте импортирует пользователей, права и топологию из `infra/rabbitmq/definitions.json`; +- Grafana подхватывает provisioning. + +**Что делает оператор после рестарта OpenBao:** `sh scripts/bao.sh unseal`. Уже работающие сервисы продолжают работать, перезапущенные ждут unseal. Restart hook (`scripts/restart.sh`) делает unseal сам, потому что его запускает оператор. + +**Проверки:** + +```sh +docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro \ + -v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py # вся матрица доверия, 135 проверок +python3 student-tests/test_auth_acl.py # ACL, X-User-Id, logout/expiry, лимит входа, размеры, rehash +python3 student-tests/test_cert_rotation.py # ротация и отзыв leaf lb-client, просроченный сертификат +python3 student-tests/check_hardening.py # docker inspect всех контейнеров +python3 student-tests/test_observability.py # трассы/логи/метрики через mTLS, отсутствие утечек +python3 student-tests/check_dashboard.py # все запросы dashboard, включая выполнение через Grafana +python3 student-tests/test_worker_down.py; python3 student-tests/test_failure_windows.py # регрессия lab5 +sh student-tests/operator_checks.sh # ОПЕРАТОР: ротация пароля worker, seal/unseal, persistence, failover +``` + +CI: `.github/workflows/stand.yml` повторяет всю последовательность оператора в одноразовом runner (PKI, OpenBao init/bootstrap, стенд, `make test`, `make test-security`, TLS-аудит, hardening). Локально эти шаги проверены; на runner workflow ещё не выполнялся. + +Для LOGIN DoS-замера: `LOGIN_RATE_LIMIT_PER_IP=1000000 docker compose --env-file .env up -d --no-deps api-1 api-2 && python3 student-tests/login_budget.py`, затем вернуть лимит обычным `up`. + +**Остановка и удаление.** Безопасная остановка с сохранением данных: `docker compose stop`. Удаление учебных данных: `docker compose down -v`. После этого OpenBao нужно инициализировать заново, а `secrets/operator/` устареет. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ──TLS :8443──► nginx ──mTLS (lb.client.messenger; API проверяет SAN и отзыв)──► api-1, api-2 + api ─AMQPS mTLS (EXTERNAL=CN, только publish)─► RabbitMQ ─► worker (только consume) + api, worker ──TLS verify-full──► PostgreSQL (роли app_api / app_worker) + api, worker ──HTTPS──► MinIO (пользователи messenger-api / messenger-worker, права по префиксам) + api ──TLS + ACL──► Valkey (кэш сессий + общий лимит входа) + api, worker ──HTTPS + cert auth──► OpenBao (KV: только свой путь, токен 15 мин) + api, worker ──mTLS (Ingest CA)──► Collector ──mTLS (Telemetry Clients CA)──► Prometheus / Loki / Tempo ◄── Grafana +оператор ──docker compose exec / 127.0.0.1 (HTTPS, mTLS)──► OpenBao, Grafana, Prometheus/Loki/Tempo +``` + +**Источник истины и транзакции** (как в lab5–lab6): +- PostgreSQL — источник истины; +- upload в одной транзакции фиксирует вложение, задание и outbox и только потом отвечает 202; +- worker публикует результат одной транзакцией; +- брокер переносит только ссылки на задания; +- OpenBao в бизнес-транзакции не входит: секреты читаются при старте и обновляются фоном. + +**Изменения относительно lab6:** + +| Область | Что сделано | Где | +| --- | --- | --- | +| PKI | Root CA и два промежуточных: Ingest (только клиенты Collector) и Telemetry Clients (только клиенты хранилищ телеметрии); leaf 90 дн., EC P-256; перевыпуск, отзыв по serial, тестовые просроченный и чужой сертификаты | `scripts/pki.sh` | +| Внешний TLS | nginx на `127.0.0.1:8443`, HSTS, TLS 1.2+ | `infra/nginx.conf` | +| LB → API | mTLS: nginx предъявляет `lb.client.messenger` и проверяет сервер по `api.internal`. API (`app/serve.py`): CERT_REQUIRED, allowlist DNS SAN, denylist отозванных serial (перечитывается на лету), проверка на каждом соединении. Plaintext-listener нет, health только `127.0.0.1:8081` | `app/serve.py`, `app/main.py` | +| Связи с хранилищами | PostgreSQL `verify-full` + `pg_hba` только `hostssl`; MinIO HTTPS; RabbitMQ AMQPS с обязательным клиентским сертификатом; Valkey TLS-only + ACL | `infra/postgres`, `infra/minio`, `infra/rabbitmq`, `infra/valkey`, `app/db.py`, `app/s3.py`, `app/broker.py`, `app/sessions.py` | +| Минимальные права | роли БД, S3-политики по префиксам, права RabbitMQ (API: write `^attachments$`; worker: read `^attachments\.process$`; configure ни у кого, топологию создаёт оператор) | `migrations/0007_service_roles.sql`, `infra/minio/policy-*.json`, `definitions.json` | +| OpenBao | server mode, хранилище `file` на volume, TLS; cert auth: роль = точный CN, токен 15 мин / max 1 ч; политики «только свой путь»; клиент с renew/re-login и подхватом новой версии KV без рестарта | `infra/openbao/`, `scripts/bao.sh`, `app/openbao.py` | +| Телеметрия | Collector принимает только Ingest CA; Prometheus, Loki и Tempo — только Telemetry Clients CA; Grafana обращается к источникам с клиентским сертификатом; scrape-заданий нет; внутренний gRPC и self-метрики только на loopback | `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` | +| Аутентификация | Argon2id с параметрами из env (замер ниже); rehash при входе; ≤ 4 одновременных Argon2 на реплику; лимит входа 5 неудач на username и 120 попыток на IP за 60 с — общий для реплик (Valkey), 429 + Retry-After | `app/passwords.py`, `app/main.py`, `app/sessions.py` | +| Hardening | все 15 контейнеров non-root, `cap_drop: ALL`, `no-new-privileges`, read-only rootfs + tmpfs, лимиты CPU/RAM/PIDs; нет Docker socket и host network; наружу только `127.0.0.1` | `compose.yaml` | +| Обновления по scan | `python:3.12.16-slim`, Pillow 13.0.0, urllib3 2.8.0, nginx 1.30.6, OpenBao 2.6.5; guard использует `scope["path"]` вместо `request.url.path` | `Dockerfile`, `requirements.txt`, `compose.yaml` | + +**Роли, версии, health, volumes, порты, credentials.** Сеть одна — внутренняя сеть Compose. Digest всех образов закреплён в `compose.yaml`. Каждый ключ попадает только в свой контейнер как Docker secret. + +| Роль | Сервис | Образ | Health | Volume | Порт (наружу) | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `nginx` | nginx:1.30.6-alpine | `curl https://localhost:8443/health/live` | — | `127.0.0.1:8443` | lb-server, lb-client | +| API ×2 | `api-1`, `api-2` | сборка, python:3.12.16-slim | loopback `127.0.0.1:8081/health/ready` | — | — | api-N (сервер), api (клиент), api-otel; **паролей нет**, всё из OpenBao | +| Worker ×2 | `worker` | та же сборка | heartbeat-файл | — | — | worker (клиент), worker-otel; паролей нет | +| БД | `postgres` | postgres:17.7-alpine | `pg_isready` | `pgdata` | — | postgres (сервер), пароль superuser (только migrate/оператор) | +| S3 | `minio` | madlabcg/minio (digest) | `mc ready` по HTTPS | `miniodata` | — | minio (сервер), root (только minio-init) | +| Брокер | `rabbitmq` | rabbitmq:4.0.5-alpine | `check_running` | `rabbitmqdata` | — | rabbitmq (сервер); пользователей с паролями нет | +| Кэш и лимиты | `valkey` | valkey:8.1-alpine | `PING` пользователем `health` | — (кэш) | — | valkey (сервер), пароль ACL `api` | +| Секреты | `openbao` | openbao:2.6.5 | `bao status` | `baodata` | — | openbao (сервер); root token и доли у оператора | +| Telemetry | `otel-collector`, `prometheus`, `loki`, `tempo` | версии lab6 | promtool / TCP | `promdata`, `lokidata`, `tempodata` | `127.0.0.1:9090/3100/3200` (mTLS operator) | свои серверные и клиентские сертификаты | +| UI | `grafana` | grafana:11.5.2 | `curl https://grafana:3000/api/health` | `grafanadata` | `127.0.0.1:3000` (HTTPS + login) | grafana, grafana-out, admin-пароль | + +**Соответствие `.env.example`:** +- `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `S3_*_FILE`, `BROKER_URL_FILE` для сервисов больше не используются: эти значения приходят из OpenBao (`messenger/api`, `messenger/worker`); +- `BROKER_URL` = `amqps://rabbitmq:5671/messenger?auth=external` — без пароля; +- `TLS_CA_FILE`, `TLS_CLIENT_CERT_FILE` и `TLS_CLIENT_KEY_FILE` (вместо `TLS_CERT_FILE`/`TLS_KEY_FILE` из примера), `TLS_SERVER_*`, `BAO_ADDR` задаются в `compose.yaml`; +- `ARGON2_*` = 19456 KiB / 2 / 1; +- добавлены `PASSWORD_HASH_CONCURRENCY`, `LOGIN_RATE_LIMIT_*`, `BAO_REFRESH_SECONDS` и `BAO_STARTUP_TIMEOUT_SECONDS`. + +**Документы:** модель угроз — [security/THREAT_MODEL.md](security/THREAT_MODEL.md) (14 угроз с тестами и разбор scan); матрица доверия — [contracts/trust-matrix.md](contracts/trust-matrix.md); инструкция оператора OpenBao — [infra/openbao/README.md](infra/openbao/README.md). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда / сценарий | Наблюдаемый результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| 3: прежние возможности через HTTPS с проверкой CA и SAN | `make check`, `make validate`, `make test`, `make test-security` | все PASS; пустой trust store → ошибка проверки сертификата | `evidence/smoke_check_validate_security.txt`, `evidence/smoke.txt` | +| 3: persistence через HTTPS | `make test-persistence` (hook перезапускает все хранилища, включая OpenBao с unseal, и сервисы) | PASS; uptime всех контейнеров сброшен | `evidence/persistence.txt` | +| 3: отказ реплики через HTTPS | `make test-failover` ×2 | PASS; 0 неудачных попыток, восстановление 1.27 и 0.67 с | `evidence/failover.txt` | +| 3: минимум lab5–lab6 | `test_worker_down.py`, `test_failure_windows.py` (A–I), `variants_check.py`, `test_observability.py` | все PASS | `evidence/lab6_worker_down.txt`, `evidence/lab5_failure_windows.txt`, `evidence/variants_limits.txt`, `evidence/observability.txt` | +| 3: Argon2id ≥ 19 MiB, t=2, p=1, параметры обоснованы замером | `login_budget.py` (bench в контейнере API, 0.5 CPU); `test_auth_acl.py` | PHC-строка `$argon2id$v=19$m=19456,t=2,p=1$…` с солью; 11 мс на хеш; таблица ниже | `evidence/login_budget.json`, `evidence/auth_acl.txt` | +| 3: LB → API mTLS, отказ без сертификата, с чужим CA, с неверной identity | аудит A | 14/14: без сертификата, чужой CA с именем lb, `intruder`/`worker`/`api` от того же CA → соединение закрыто; `lb` → 200; plaintext → нет ответа | `evidence/tls_audit_final.txt` | +| 3: нет обхода публичным портом | аудит A, K; `check_hardening.py` | порты API не опубликованы; health 8081 закрыт из сети | `evidence/tls_audit_side_channels.txt`, `evidence/hardening.txt` | +| 3: TLS с проверкой сервера на всех связях, раздельные учётные данные | аудит B, C, D, E, F, G, H, I, J | чужой CA как trust anchor, неверный hostname, plaintext → отказ на каждой связи | `evidence/tls_audit_final.txt` | +| 3: OpenBao не dev-mode, persistent, сервисная аутентификация, scoped policy, без root token в app | аудит H; `bao.sh`; persistence | cert auth: только своя роль, токен ≤ 15 мин; чужой путь, запись, политика, admin-пути → 403; данные пережили рестарт OpenBao | `evidence/tls_audit_final.txt`, `evidence/persistence.txt` | +| 3: контейнеры без privileged/host network/socket, non-root, без лишних capabilities, лимиты; наружу только нужное | `check_hardening.py` (`docker inspect`) | 15 контейнеров, 0 проблем; опубликованы только 8443, 3000, 9090, 3100, 3200 на 127.0.0.1 | `evidence/hardening.txt`, `evidence/hardening.json` | +| 3: секреты не в Git, образах, telemetry | Trivy secret scan; `test_observability.py` | в слоях образа секретов 0; пароль, токен, текст, filename, Authorization, PNG в трассах и логах не найдены | `evidence/image_scan.txt`, `evidence/observability.txt` | +| 3: лимит входа 429 + Retry-After, размер запросов, посторонний, logout/expiry | `test_auth_acl.py` | 404 постороннему на всё; поддельный X-User-Id игнорируется; logout и expiry → 401 на обеих репликах; 70 kB JSON и > 10 MiB upload → 413; лимит входа → 429 | `evidence/auth_acl.txt` | +| 3: локальный bootstrap сертификатов | `scripts/pki.sh init` | CA и leaf создаются в `secrets/pki/`, ключи не в Git | `scripts/pki.sh` | +| 4: mTLS к брокеру и Collector, разные identities, чужой сертификат того же CA не даёт прав | аудит D, E | RabbitMQ: `intruder`, `lb` (тот же CA) → ACCESS_REFUSED; API не может consume и declare; worker не может publish. Collector: `lb`, `api` (Root), `grafana-out` → отказ; api-otel, worker-otel → 200 | `evidence/tls_audit_final.txt` | +| 4: хеши и ошибки не раскрывают credentials; rehash; бюджет login DoS; общий rate limit | `test_auth_acl.py`, `login_budget.py` | старый хеш `m=8192,t=1` пересчитан в `m=19456,t=2` при входе; 5 неудач на api-1 и api-2 вместе → 429 даже с верным паролем; тело 429 без деталей | `evidence/auth_acl.txt`, `evidence/login_budget.json` | +| 4: ротация одного рабочего secret без пересборки | `test_secret_rotation.py` (оператор) | пароль `app_worker` изменён в PostgreSQL и OpenBao; оба worker переключились за 10 с; после принудительного переподключения задания обрабатываются; старый пароль → `password authentication failed`; 0 ошибок, 0 упавших заданий | `evidence/secret_rotation.txt` | +| 4: ротация leaf без пересборки, отзыв старой identity | `test_cert_rotation.py` | `lb-client` перевыпущен (serial …1418 → …142A), `nginx -s reload` за 1.4 с; после отзыва старый сертификат закрыт на обеих репликах (`reason=revoked`); 161 запрос, 0 ошибок | `evidence/cert_rotation.txt` | +| 4: ограниченное поведение при expiry и sealed OpenBao | `test_bao_sealed.py` (оператор), `test_cert_rotation.py` | sealed: smoke проходит, сервисы держат секреты, перезапущенный worker ждёт и не стартует без секретов, после unseal здоров через 16 с. Просроченный сертификат с верной identity отклоняется | `evidence/bao_sealed.txt`, `evidence/cert_rotation.txt` | +| 4: bootstrap trust и renewal описаны, нет постоянного «секрета для секретов» | `app/openbao.py`, модель угроз | первичная identity — TLS-ключ сервиса; токен 15 мин, renew/re-login каждые 15 с | `security/THREAT_MODEL.md`, `infra/openbao/README.md` | +| 4: read_only, no-new-privileges, drop capabilities, tmpfs, исключения объяснены | `check_hardening.py` | read-only rootfs у **всех** 15 контейнеров, включая stateful; запись только в volumes и tmpfs (для non-root nginx tmpfs смонтированы с его uid). Исключений не понадобилось | `evidence/hardening.txt` | +| 4: dependency/image scan с разбором применимости | Trivy по образу приложения, nginx, OpenBao | приложение: CRITICAL ОС 9 → 2 (исправлений нет, неприменимы), Python CRITICAL 5 → 0, HIGH 43 → 3 (starlette, неприменимы); nginx и OpenBao обновлены | `evidence/image_scan.txt`, `security/THREAT_MODEL.md` (раздел Scan) | +| 4: ≥ 6 угроз с контролями и воспроизводимыми тестами | модель угроз | 14 угроз T1–T14, у каждой отрицательный тест | `security/THREAT_MODEL.md` | +| 4: upload-worker ограничен по CPU/RAM/времени | `variants_check.py` | дочерний процесс с RLIMIT_AS/CPU и таймаутом; изображение 39.5 Мпикс — пик RSS 337 MiB; лимиты пикселей соблюдены | `evidence/variants_limits.txt` | +| 4: TLS-аудит всей матрицы, ACL, отсутствие утечек | `tls_audit.py`, `test_auth_acl.py`, `test_observability.py` | 135 проверок, 0 провалов (A–K); ACL и приватность — выше | `evidence/tls_audit_final.txt` | +| Внутренняя телеметрия и Grafana работают через mTLS | `check_dashboard.py` | 32 панели; все 44 запроса Prometheus, Loki и Tempo выполнены через Grafana | `evidence/dashboard_check.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: macOS arm64, Docker 29.7.2, VM 10 CPU / 7.7 GiB. Лимиты: API 0.5 CPU / 512 MiB ×2, worker 2 CPU / 1.5 GiB ×2, PostgreSQL 1 CPU / 512 MiB, OpenBao 0.5 CPU / 256 MiB, nginx 0.25 CPU / 64 MiB. Версии — в таблице ролей. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +### Параметры Argon2id (замер в контейнере API, лимит 0.5 CPU, 10 хешей) + +| m, KiB | t | p | p50, мс | max, мс | Пик RSS, MiB | +| --- | --- | --- | --- | --- | --- | +| **19456** | **2** | **1** | **11.3** | 66.6 | 37 | +| 12288 | 3 | 1 | 9.3 | 62.0 | 37 | +| 47104 | 1 | 1 | 16.6 | 69.2 | 83 | +| 65536 | 3 | 1 | 171.6 | 175.5 | 101 | +| 65536 | 3 | 4 | 390.0 | 806.1 | 101 | + +**Выбор: m = 19 MiB, t = 2, p = 1** (минимум OWASP). +- При 4 параллельных слотах на реплику это ~80 MiB памяти и ~11 мс CPU на вход. Обе реплики выдерживают ~40 входов/с, оставаясь в лимите 512 MiB. +- Вариант `m = 64 MiB, t = 3` дороже в 15 раз по времени: 4 слота заняли бы ~400 MiB из 512 (риск OOM), а пропускная способность входа упала бы до ~3/с на реплику. Это сделало бы DoS через вход проще. +- Замер — на этом ноутбуке. CFS разрешает кратковременно превышать квоту CPU, поэтому под длительной нагрузкой время растёт (см. ниже). + +### Бюджет защиты от DoS через вход (`login_budget.py`, 30 с на фазу) + +Модель распределённого атакующего: лимит по IP поднят на время замера, у каждой попытки новый несуществующий username. Параллельно идут обычные запросы (POST сообщения, 5 в секунду). + +| Фаза | Попыток входа/с | Латентность 401, p50 / p99 | CPU обеих API, % ядра | RSS API | Обычные запросы: ошибки, p50 / p99 | +| --- | --- | --- | --- | --- | --- | +| без атаки | 0 | — | 10.3 | 97 MiB | 0, 9.5 / 46.4 мс | +| 32 атакующих | 40.4 | 803 / 2397 мс | 89.6 (лимит 100) | 183–197 MiB | 0, 28.5 / 295 мс | + +Вывод: атака упирается в лимит CPU, но память ограничена слотами Argon2 (рост на ~100 MiB), а обычный трафик продолжает обслуживаться без ошибок, хотя p99 растёт в 6 раз. Отказов 503 «hash_busy» не было, очередь к слоту укладывалась в 3 с. С одного IP атакующий получает 429 после 107–120 попыток в минуту (`auth_acl.txt`). + +### Отказы и ротации + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потери | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Ротация leaf `lb-client` под нагрузкой 10 запросов/с | без пересборки и простоя; старый отозван | `pki.sh issue` → `nginx -s reload` (1.4 с) → старый и новый действуют → `revoke-serial` → старый отклонён на api-1 и api-2 через ~1 с, новый 200, просроченный отклонён | 161/161 запросов успешно, smoke после — PASS | +| Ротация пароля `app_worker` под трафиком (5 запросов/с + 1 upload/с) | worker переключается без рестарта, старый пароль недействителен | новая версия KV подхвачена обоими worker за 10 с; сессии `app_worker` принудительно завершены, переподключение с новым паролем; 3 upload → ready | 116 запросов, 20 upload, 0 ошибок, 0 упавших заданий | +| OpenBao запечатан на 58 с | работающие сервисы продолжают, новые ждут, ничего не ослабляется | smoke PASS при запечатанном OpenBao; `openbao_unavailable` в логах; перезапущенный worker-2 в состоянии `starting`, ждёт OpenBao; upload обработан вторым worker | после unseal worker-2 healthy через 16 с | +| Рестарт всех хранилищ, включая OpenBao (hook persistence) | данные и сессии сохраняются | PASS; OpenBao распечатан оператором внутри hook | потерь нет | +| Остановка api-1, затем api-2 (failover) | запись восстанавливается ≤ 10 с, без дублей | PASS ×2: 128 и 130 записей, 0 неудачных попыток | 1.27 и 0.67 с | +| Обновление OpenBao 2.1.0 → 2.6.5 (по scan) | хранилище `file` подхватывается | версия 2.6.5 стартовала на старом volume, данные на месте; после unseal вход сервисов работает | требуется unseal (как после любого рестарта) | + +### Что нашли и исправили по ходу проверок + +- **Valkey ACL** не разрешал `INCRBY`, хотя valkey-py вызывает её для `INCR`: общий лимит входа молча откатился бы на локальный. Найдено аудитом G. +- **Erlang distribution RabbitMQ** (25672) и **кластерный порт OpenBao 2.6** (8201) были доступны из сети. Привязаны к loopback и отключены соответственно (аудит K). +- **Self-метрики Collector** нельзя было отправить по OTLP с клиентским сертификатом. Переведены на loopback-сбор внутри процесса, plaintext-scrape убран совсем. +- **Self-метрики Collector** приходили с `service.instance.id` = случайный UUID (новый после каждого рестарта), это давало рост кардинальности меток. Задано стабильное значение. +- **У worker не было лимита PIDs** (`check_hardening.py`). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Выдерживается** (проверено): отказ реплики API; рестарт всех хранилищ с volumes; запечатанный или перезапущенный OpenBao (без простоя работающих сервисов); недоступная телеметрия (из lab6); компрометация отдельной identity — её права ограничены (аудит), сертификат LB отзывается. + +**Не выдерживается и не заявляется:** потеря Docker-host и компрометация хоста или Docker daemon (root на хосте читает `secrets/`); высокая доступность единственных PostgreSQL, nginx, OpenBao и Collector. + +**Учебные упрощения и известные ограничения:** +- Ключи CA, все 3 доли unseal и root token — у одного оператора на хосте стенда (`secrets/`, вне Git, 0600/0700). В production — разные держатели, HSM/KMS, отзыв root token. +- Отзыв по serial проверяет только API. RabbitMQ, OpenBao и Collector отзыв не проверяют; там identity отзывается удалением пользователя или роли. +- Collector не связывает `service.name` с сертификатом. В Prometheus, Loki и Tempo нет прав по identity: любой клиент Telemetry Clients CA и пишет, и читает (CA выдан только collector, grafana, operator). +- Истечение токена OpenBao (15 мин) не выжидалось в тесте. Продление и повторный вход видны в логах (`openbao_login`), истечение сертификата проверено просроченным сертификатом. +- Алерты lab6 (сценарии `test_incidents.py`) в lab7 заново не прогонялись. Проверены трассы, логи, приватность и все запросы dashboard через mTLS. Правила алертов загружены (`promtool check rules`: 13 правил). +- Исчерпание пула БД переводит реплику в not ready (из lab6). Отдельный пул для readiness не реализован. +- `FAULT_INJECTION=1` и `X-Fault` допустимы только на учебном стенде. TCP-healthcheck Loki и Tempo оставляет в их логах записи `TLS handshake error … EOF`. +- Starlette 0.41.3 не обновлялась: применимых уязвимостей не осталось, а обновление требует мажорного перехода FastAPI. +- Первый запуск PKI и OpenBao — ручные шаги оператора; они воспроизводимы скриптами. + +**Следующие шаги:** короткоживущие сертификаты из OpenBao PKI с автоматическим renewal (трек 2), проверка отзыва во всех сервисах (CRL/OCSP), отдельный пул для readiness, обновление FastAPI и Starlette. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 0000000..0c21ba2 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,81 @@ +"""RabbitMQ topology and confirmed publishing (aio-pika). + +Topology (lab7: declared by the operator in infra/rabbitmq/definitions.json; services only check it passively): + exchange attachments direct, durable + queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit + exchange attachments.dlx fanout, durable + queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator) +Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker +has taken responsibility for the message (written to the quorum log). +""" +import json +import logging +import ssl + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType, Message + +from app import config + +log = logging.getLogger("messenger.broker") +ROUTING_KEY = "attachment.process.v1" + + +def tls_context() -> ssl.SSLContext: + """lab7: AMQPS with mutual TLS. The broker verifies our client certificate and maps its CN to the RabbitMQ user + (auth=external): api.client.messenger may only publish, worker.client.messenger may only consume.""" + ctx = ssl.create_default_context(cafile=config.TLS_CA_FILE) + ctx.load_cert_chain(config.TLS_CLIENT_CERT_FILE, config.TLS_CLIENT_KEY_FILE) + return ctx + + +async def connect(): + return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, ssl_context=tls_context(), + client_properties={"connection_name": config.INSTANCE_ID}) + + +async def get_exchange(channel): + """Passive: the topology (exchanges, quorum queues, DLX, bindings) is created by the operator from + infra/rabbitmq/definitions.json. Services have NO configure permission, so they only check that it exists.""" + return await channel.get_exchange(config.BROKER_EXCHANGE, ensure=True) + + +async def get_queue(channel): + return await channel.get_queue(config.BROKER_QUEUE, ensure=True) + + +class Publisher: + def __init__(self) -> None: + self.connection = None + self.channel = None + self.exchange = None + + async def ensure(self) -> None: + if self.channel is not None and not self.channel.is_closed: + return + if self.connection is None or self.connection.is_closed: + self.connection = await connect() + self.channel = await self.connection.channel(publisher_confirms=True) + self.exchange = await get_exchange(self.channel) + + async def publish(self, body: dict) -> None: + await self.ensure() + message = Message( + json.dumps(body, separators=(",", ":")).encode(), + content_type="application/json", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=body["job_id"], + type=body["type"], + # Native AMQP headers duplicate the body carrier (same W3C values) for broker-side tooling. + headers={"attempt": body["attempt"], **{k: body[k] for k in ("traceparent", "tracestate") if k in body}}, + ) + # mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack. + await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5) + + async def close(self) -> None: + try: + if self.connection is not None: + await self.connection.close() + except Exception: + pass + self.connection = self.channel = self.exchange = None diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..a3741b0 --- /dev/null +++ b/app/config.py @@ -0,0 +1,134 @@ +import os +import socket + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + path = os.environ.get(name + "_FILE") + if path: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + return os.environ.get(name, default) + + +# --- lab7: TLS material (paths only) and secrets from OpenBao ------------------------------------------------ +TLS_CA_FILE = os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt") +TLS_CLIENT_CERT_FILE = os.environ.get("TLS_CLIENT_CERT_FILE", "/run/secrets/client.crt") +TLS_CLIENT_KEY_FILE = os.environ.get("TLS_CLIENT_KEY_FILE", "/run/secrets/client.key") +DATABASE_SSLMODE = os.environ.get("DATABASE_SSLMODE", "verify-full") +BAO_REFRESH_SECONDS = _float("BAO_REFRESH_SECONDS", 15.0) +SECRETS = {} +if os.environ.get("BAO_ADDR"): + from app import openbao as _openbao + + SECRETS = _openbao.load(_float("BAO_STARTUP_TIMEOUT_SECONDS", 120.0)) + + +def _from_bao(key, env_name, default=""): + """OpenBao value when the service runs with BAO_ADDR, otherwise the old *_FILE / env (tools, tests).""" + return SECRETS.get(key) or _secret(env_name, default) + + +DATABASE_URL = _from_bao("database_url", "DATABASE_URL") +CACHE_URL = _from_bao("cache_url", "CACHE_URL") +SESSION_STORE = os.environ.get("SESSION_STORE", "postgres") +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30) +SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60) +DB_POOL_MIN = _int("DB_POOL_MIN", 1) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0) +DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3) +DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000) +DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0) +REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0) +DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5) +READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0) +READY_FAILURES = _int("READY_FAILURES", 2) +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2) +MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64) +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_RSS_MB = _int("MAX_RSS_MB", 0) +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 +SESSION_REVOKED_TTL_SECONDS = max( + _int("SESSION_REVOKED_TTL_SECONDS", 60), + SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5, +) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _from_bao("s3_access_key_id", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _from_bao("s3_secret_access_key", "S3_SECRET_ACCESS_KEY") +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760) +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000) +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0) +UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576) +RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300) +RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2) + +# --- lab5: broker, jobs, worker --- +BROKER_URL = _secret("BROKER_URL") # amqps://...?auth=external: identity = client certificate, no password +BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments") +BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx") +BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq") +BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20) +PIPELINE_VERSION = _int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) +WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY +JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60) +JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20) +JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0) +JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0) +JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0) +WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768) +WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25) +WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat") +OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0) +OUTBOX_BATCH = _int("OUTBOX_BATCH", 50) +OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400) +STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400) +# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables). +MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200) +PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0) +THUMBNAIL_SIZE = 256 +CROP_SIZE = 128 +OPTIMIZED_SIZE = 1280 +JPEG_QUALITY = _int("JPEG_QUALITY", 85) +WEBP_QUALITY = _int("WEBP_QUALITY", 80) +# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py). +FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1" + +# --- lab7: authentication hardening --------------------------------------------------------------------------- +ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) +ARGON2_ITERATIONS = _int("ARGON2_ITERATIONS", 2) +ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1) +PASSWORD_HASH_CONCURRENCY = _int("PASSWORD_HASH_CONCURRENCY", 4) +PASSWORD_HASH_QUEUE_SECONDS = _float("PASSWORD_HASH_QUEUE_SECONDS", 3.0) +LOGIN_RATE_LIMIT_PER_USER = _int("LOGIN_RATE_LIMIT_PER_USER", 5) +LOGIN_RATE_LIMIT_PER_IP = _int("LOGIN_RATE_LIMIT_PER_IP", 120) +LOGIN_RATE_WINDOW_SECONDS = _int("LOGIN_RATE_WINDOW_SECONDS", 60) +# mTLS on the API listener: only these client identities (DNS SAN) may connect; revoked serials are refused. +ALLOWED_CLIENT_IDENTITIES = frozenset( + x.strip() for x in os.environ.get("ALLOWED_CLIENT_IDENTITIES", "lb.client.messenger").split(",") if x.strip()) +REVOKED_SERIALS_FILE = os.environ.get("REVOKED_SERIALS_FILE", "") +TLS_SERVER_CERT_FILE = os.environ.get("TLS_SERVER_CERT_FILE", "") +TLS_SERVER_KEY_FILE = os.environ.get("TLS_SERVER_KEY_FILE", "") +HEALTH_LISTEN_PORT = _int("HEALTH_LISTEN_PORT", 8081) +GRACEFUL_TIMEOUT = _int("GRACEFUL_TIMEOUT", 8) diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 0000000..113aabc --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +import uuid +from datetime import datetime, timezone + +TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") +TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class BadCursor(Exception): + pass + + +def fmt(ts: datetime) -> str: + return ts.astimezone(timezone.utc).strftime(TS_FORMAT) + + +def encode_cursor(scope: str, created_at: str, item_id: str) -> str: + raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(scope: str, cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + data = json.loads(raw) + except Exception: + raise BadCursor() from None + if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope: + raise BadCursor() + ts, item_id = data["t"], data["i"] + if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str): + raise BadCursor() + try: + parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc) + if str(uuid.UUID(item_id)) != item_id: + raise BadCursor() + except ValueError: + raise BadCursor() from None + if encode_cursor(scope, ts, item_id) != cursor: + raise BadCursor() + return parsed, item_id diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..7d925af --- /dev/null +++ b/app/db.py @@ -0,0 +1,145 @@ +import asyncio +import logging +import re +import time +from contextlib import asynccontextmanager + +import psycopg +from opentelemetry.trace import SpanKind, Status, StatusCode +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from app import config, telemetry +from app.errors import unavailable + +BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) +OPERATION_RE = re.compile(r"^\s*([A-Za-z]+)") +TABLE_RE = re.compile(r"\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-z_]+)", re.IGNORECASE) +POOL_WAIT_SPAN_SECONDS = 0.005 +log = logging.getLogger("messenger.db") + + +def statement(query) -> tuple: + """Span name and attributes for a parametrised SQL statement (values are never part of the text).""" + text = query if isinstance(query, str) else str(query) + op = OPERATION_RE.match(text) + table = TABLE_RE.search(text) + operation = op.group(1).upper() if op else "SQL" + name = operation + (" " + table.group(1) if table else "") + return name, {"db.system": "postgresql", "db.operation.name": operation, "db.namespace": "messenger", + "db.collection.name": table.group(1) if table else "", "db.query.text": text[:400]} + + +async def discard(conn) -> None: + try: + await conn.close() + except Exception: + pass + + +class Guarded: + def __init__(self, conn, timeout: float): + self.conn = conn + self.timeout = timeout + + async def execute(self, *args, **kwargs): + # Own CLIENT span around the awaited statement: the time really spent in PostgreSQL (lock waits included). + # (opentelemetry-instrumentation-psycopg 0.51b0 ends async spans before the await completes -> 0 ms spans.) + name, attrs = statement(args[0] if args else "") + with telemetry.span(name, kind=SpanKind.CLIENT, attributes=attrs) as span: + try: + return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout) + except asyncio.TimeoutError: + await discard(self.conn) + span.set_status(Status(StatusCode.ERROR, "client timeout %.1fs" % self.timeout)) + raise unavailable("Хранилище не ответило за отведённое время") from None + + def transaction(self): + return self.conn.transaction() + + +class Database: + def __init__(self) -> None: + options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % ( + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS * 2, + ) + self.pool = AsyncConnectionPool( + conninfo=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_POOL_TIMEOUT, + max_lifetime=600.0, + open=False, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": config.DB_CONNECT_TIMEOUT, + "options": options, + # lab7: TLS with full server verification (CA + hostname "postgres"); no fallback to plaintext. + "sslmode": config.DATABASE_SSLMODE, + "sslrootcert": config.TLS_CA_FILE, + }, + ) + + def rotate(self, conninfo: str) -> None: + """New credentials from OpenBao: new connections use them; open ones keep working until max_lifetime.""" + if conninfo and conninfo != self.pool.conninfo: + self.pool.conninfo = conninfo + log.info("db_credentials_rotated: new connections use the new password") + + async def open(self) -> None: + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close() + + @asynccontextmanager + async def connection(self): + started, acquired = time.time_ns(), False + try: + async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn: + acquired = True + waited = (time.time_ns() - started) / 1e9 + if waited >= POOL_WAIT_SPAN_SECONDS: # only noticeable waits become spans + telemetry.tracer.start_span("db.pool.acquire", start_time=started, + attributes={"db.system": "postgresql", + "db.client.connection.wait_s": round(waited, 4)}).end() + yield Guarded(conn, config.DB_OP_TIMEOUT) + except BROKEN as exc: + if not acquired: + span = telemetry.tracer.start_span("db.pool.acquire", start_time=started, + attributes={"db.system": "postgresql"}) + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + span.end() + raise unavailable("Хранилище недоступно") from None + + @asynccontextmanager + async def transaction(self): + async with self.connection() as conn: + async with conn.transaction(): + yield conn + + async def in_transaction(self, action): + async def run(): + async with self.transaction() as conn: + return await action(conn) + + try: + return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT) + except asyncio.TimeoutError: + raise unavailable("Хранилище не ответило за отведённое время") from None + + async def ready(self) -> bool: + try: + async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn: + try: + await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT) + except asyncio.TimeoutError: + await discard(conn) + return False + return True + except Exception: + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 0000000..8110d27 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +from typing import Optional + +from fastapi.responses import JSONResponse + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(scheme: str = "Bearer") -> ApiError: + challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE + message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия" + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Объект не найден") + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message, {"Retry-After": "1"}) + + +def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})}, + ) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 0000000..d9a2dfc --- /dev/null +++ b/app/faults.py @@ -0,0 +1,38 @@ +"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off). + +A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`. +Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata. +Crash faults fire only on the first attempt and first delivery, so recovery can be observed. + api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish + worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT + worker_crash_after_commit worker exits after the result COMMIT, before broker ack + worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1 + transientN attempts 1..N fail with a transient error (N = 1..9) + permanent permanent error on the first attempt -> failed without retries +""" +import logging +import os +import re +from typing import Optional + +from app import config + +log = logging.getLogger("messenger.faults") +TRANSIENT_RE = re.compile(r"fault-transient([1-9])") + + +def active(filename: Optional[str], name: str) -> bool: + return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename + + +def transient_until(filename: Optional[str]) -> int: + if not config.FAULT_INJECTION or not filename: + return 0 + match = TRANSIENT_RE.search(filename) + return int(match.group(1)) if match else 0 + + +def crash(point: str) -> None: + log.error("FAULT INJECTION: process exits at %s", point) + logging.shutdown() + os._exit(70) diff --git a/app/imagecheck.py b/app/imagecheck.py new file mode 100644 index 0000000..fe698c5 --- /dev/null +++ b/app/imagecheck.py @@ -0,0 +1,213 @@ +"""Integrity checks for uploaded PNG/JPEG with bounded memory. + +PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in +0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy +stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is +only a warning. Both finish before anything is stored. +""" +import logging +import os +import struct +import subprocess +import threading +import time +import zlib + +from PIL import Image + +from app import config +from app.errors import ApiError, unavailable + +log = logging.getLogger("messenger.imagecheck") + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +Image.MAX_IMAGE_PIXELS = None +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +FULL_DECODE_MAX_PIXELS = 4000000 +SCAN_SECONDS = 10.0 +OUT_STEP = 1 << 20 +# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit. +DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null") +jpeg_slot = threading.Lock() + + +class Invalid(Exception): + pass + + +def invalid() -> ApiError: + return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG") + + +class RowPlan: + def __init__(self, segments): + self.segments = [s for s in segments if s[0] > 1 and s[1] > 0] + self.seg = 0 + self.done = 0 + self.pos = 0 + + def feed(self, data: bytes) -> None: + i, n = 0, len(data) + while i < n: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + if self.seg >= len(self.segments): + raise Invalid("data after the last row") + length, rows = self.segments[self.seg] + if self.pos > 0: + take = min(length - self.pos, n - i) + i += take + self.pos += take + if self.pos == length: + self.pos = 0 + self.done += 1 + continue + full = min((n - i) // length, rows - self.done) + if full > 0: + if max(data[i:i + full * length:length]) > 4: + raise Invalid("bad filter type") + i += full * length + self.done += full + else: + if data[i] > 4: + raise Invalid("bad filter type") + self.pos = n - i + i = n + + def complete(self) -> bool: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + return self.seg >= len(self.segments) and self.pos == 0 + + +def png_plan(width, height, depth, color, interlace): + bits = CHANNELS[color] * depth + if not interlace: + return RowPlan([(1 + (width * bits + 7) // 8, height)]) + segments = [] + for x0, y0, dx, dy in ADAM7: + w = (width - x0 + dx - 1) // dx if width > x0 else 0 + h = (height - y0 + dy - 1) // dy if height > y0 else 0 + if w and h: + segments.append((1 + (w * bits + 7) // 8, h)) + return RowPlan(segments) + + +def scan_png(fileobj) -> None: + deadline = time.monotonic() + SCAN_SECONDS + fileobj.seek(0) + if fileobj.read(8) != PNG_SIGNATURE: + raise Invalid("signature") + plan = decomp = None + palette = seen_idat = idat_closed = False + color = None + first = True + while True: + if time.monotonic() > deadline: + raise Invalid("scan too slow") + head = fileobj.read(8) + if len(head) < 8: + raise Invalid("truncated before IEND") + length, kind = struct.unpack(">I4s", head) + if length > 0x7FFFFFFF: + raise Invalid("chunk length") + if first and kind != b"IHDR": + raise Invalid("IHDR must be first") + crc = zlib.crc32(kind) + if kind == b"IHDR": + body = fileobj.read(length) + if not first or length != 13 or len(body) != 13: + raise Invalid("IHDR") + width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1): + raise Invalid("IHDR fields") + if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]: + raise Invalid("color type / depth") + plan = png_plan(width, height, depth, color, interlace) + decomp = zlib.decompressobj() + crc = zlib.crc32(body, crc) + else: + remaining = length + while remaining: + buf = fileobj.read(min(remaining, 65536)) + if not buf: + raise Invalid("truncated chunk") + remaining -= len(buf) + crc = zlib.crc32(buf, crc) + if kind == b"IDAT": + if idat_closed or (color == 3 and not palette): + raise Invalid("IDAT order") + seen_idat = True + out = decomp.decompress(buf, OUT_STEP) + while True: + plan.feed(out) + if not decomp.unconsumed_tail: + break + if time.monotonic() > deadline: + raise Invalid("scan too slow") + out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP) + if kind == b"PLTE": + palette = True + if color in (0, 4) or length % 3 or not 3 <= length <= 768: + raise Invalid("PLTE") + elif kind != b"IDAT" and seen_idat: + idat_closed = True + tail = fileobj.read(4) + if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF: + raise Invalid("crc") + first = False + if kind == b"IEND": + if length or not seen_idat: + raise Invalid("IEND") + break + if not decomp.eof or decomp.unused_data or not plan.complete(): + raise Invalid("pixel data does not match IHDR") + + +def check_jpeg(fileobj) -> None: + fileobj.flush() + descriptor = fileobj.fileno() + os.lseek(descriptor, 0, os.SEEK_SET) + try: + with jpeg_slot: + done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=SCAN_SECONDS, check=False) + except subprocess.TimeoutExpired: + log.error("jpeg_check_timeout") + raise unavailable("Проверка изображения не успела завершиться") from None + # A signal means the checker died (e.g. OOM killer), which says nothing about the file. + if done.returncode < 0: + log.error("jpeg_check_killed signal=%d", -done.returncode) + raise unavailable("Проверка изображения прервана") + if done.returncode != 0: + raise invalid() + + +def inspect_image(fileobj, size: int, content_type: str) -> None: + """Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type).""" + try: + fileobj.seek(0) + img = Image.open(fileobj) + width, height = img.size + if img.format != FORMATS[content_type]: + raise invalid() + if width * height > config.MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей") + if content_type == "image/png": + scan_png(fileobj) + if width * height <= FULL_DECODE_MAX_PIXELS: + fileobj.seek(0) + Image.open(fileobj).load() + else: + check_jpeg(fileobj) + except ApiError: + raise + except Exception: + raise invalid() from None + finally: + fileobj.seek(0) diff --git a/app/imageproc.py b/app/imageproc.py new file mode 100644 index 0000000..13842a6 --- /dev/null +++ b/app/imageproc.py @@ -0,0 +1,115 @@ +"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a +wall-clock timeout, so a hostile or huge image can only kill this process, never the worker. + +usage: python -m app.imageproc +stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex. + +Rules (contracts/README.md "Фоновая обработка"): + thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px + crop JPEG, centred square scaled to 128x128 (small sources are upscaled) + optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4 +EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha). +EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB). +""" +import json +import math +import os +import sys + +from PIL import Image, ImageFile, ImageOps + +Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding +ImageFile.LOAD_TRUNCATED_IMAGES = False + +THUMB, CROP, OPTIMIZED = 256, 128, 1280 +JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85")) +WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80")) +ORIENTATION_TAG = 0x0112 +EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5 + + +def fit(width: int, height: int, bound: int): + """Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales.""" + if width <= bound and height <= bound: + return width, height + if width >= height: + return bound, max(1, height * bound // width) + return max(1, width * bound // height), bound + + +def has_alpha(img) -> bool: + return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info) + + +def to_8bit(img): + if img.mode in ("I;16", "I;16B", "I;16L", "I"): + return img.convert("I").point(lambda v: v * (1 / 257)).convert("L") + return img + + +def on_white(img): + if has_alpha(img): + rgba = img.convert("RGBA") + flat = Image.new("RGB", rgba.size, (255, 255, 255)) + flat.paste(rgba, mask=rgba.getchannel("A")) + return flat, rgba + return img.convert("RGB"), None + + +def process(source: str, out_dir: str, max_pixels: int) -> list: + img = Image.open(source) + if img.format not in ("PNG", "JPEG"): + raise ValueError("format") + width, height = img.size + if width * height > max_pixels: + sys.exit(EXIT_TOO_LARGE) + orientation = img.getexif().get(ORIENTATION_TAG, 1) + upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height) + thumb_size = fit(*upright, THUMB) + opt_size = fit(*upright, OPTIMIZED) + if img.format == "JPEG": + # DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept). + need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0]) + img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1]))) + img.load() + img = ImageOps.exif_transpose(img) + img = to_8bit(img) + flat, rgba = on_white(img) + del img + + results = [] + + def save(name, image, ext, fmt, content_type, **params): + path = os.path.join(out_dir, name + "." + ext) + image.save(path, fmt, **params) + results.append({"name": name, "file": path, "content_type": content_type, + "width": image.size[0], "height": image.size[1]}) + + jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False} + save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0), + "jpg", "JPEG", "image/jpeg", **jpeg) + w, h = flat.size + side = min(w, h) + left, top = (w - side) // 2, (h - side) // 2 + save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)), + "jpg", "JPEG", "image/jpeg", **jpeg) + source_for_webp = rgba if rgba is not None else flat + optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize( + opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0) + save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4) + return results + + +def main() -> None: + source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3]) + try: + results = process(source, out_dir, max_pixels) + except MemoryError: + sys.exit(EXIT_TOO_COMPLEX) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): + sys.exit(EXIT_INVALID) + json.dump(results, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 0000000..2bfa24d --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,340 @@ +"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker. + +State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it +(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes +`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT. + + enqueue nothing -> queued (API, same txn as attachment 'queued') + claim queued(due) -> processing, lease (worker; also takes over an expired lease) + heartbeat processing: lease_until += lease (only while owner is still alive) + complete processing -> succeeded + variants (fenced by lease_owner) + fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts) + sweep_leases processing(expired lease) -> queued | failed + replay failed -> queued (explicit operator action, audited) +""" +import json +import random +import uuid +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +from app import config, telemetry +from app.cursor import fmt + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 +VARIANT_ORDER = ("thumbnail", "crop", "optimized") +REQUIRED_VARIANTS = VARIANT_ORDER + + +def backoff_seconds(attempt: int) -> float: + """Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd).""" + ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict: + return { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": fmt(created_at), + } + + +def parse_message(body: bytes) -> Optional[dict]: + """Strict check against jobs.schema.json; anything else goes to the DLQ untouched.""" + try: + data = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + return None + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}: + return None + if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE: + return None + if "traceparent" in data and not (isinstance(data["traceparent"], str) + and telemetry.TRACEPARENT_RE.match(data["traceparent"])): + return None + if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512): + return None + for key in ("pipeline_version", "attempt"): + if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1: + return None + try: + data["job_id"] = str(uuid.UUID(data["job_id"])) + data["attachment_id"] = str(uuid.UUID(data["attachment_id"])) + except (ValueError, AttributeError, TypeError): + return None + return data + + +async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None): + await conn.execute( + "INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)" + " VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())", + (job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})), + ) + + +async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0): + body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"]) + # W3C trace context of the transaction that schedules the job (upload request span or the failed consumer + # span for a retry) travels in the job message (jobs.schema.json traceparent/tracestate). + body.update(telemetry.inject()) + await conn.execute( + "INSERT INTO outbox (job_id, payload, available_at, created_at)" + " VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())", + (job["id"], json.dumps(body), delay), + ) + + +async def enqueue(conn, attachment_id: str, actor: str) -> dict: + cur = await conn.execute( + "INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts," + " next_attempt_at, created_at, updated_at)" + " VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())" + " RETURNING id, attachment_id, pipeline_version, attempt, created_at", + (attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS), + ) + job = await cur.fetchone() + await _outbox(conn, job, 1) + await audit(conn, job["id"], attachment_id, "enqueued", actor, 1) + return job + + +async def pending_count(conn) -> int: + cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')") + return (await cur.fetchone())["n"] + + +async def queue_stats(conn) -> dict: + """Backlog for metrics: queued / processing jobs, age of the oldest unfinished job, unpublished outbox rows.""" + cur = await conn.execute( + "SELECT count(*) FILTER (WHERE state = 'queued') AS queued," + " count(*) FILTER (WHERE state = 'processing') AS processing," + " coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)::float8 AS oldest_age" + " FROM attachment_jobs WHERE state IN ('queued', 'processing')" + ) + row = dict(await cur.fetchone()) + cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL") + row["outbox"] = (await cur.fetchone())["n"] + return row + + +# ---------------------------------------------------------------- worker side + +JOB_COLUMNS = ( + "j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until," + " j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired," + " (j.next_attempt_at <= clock_timestamp()) AS due" +) + + +async def claim(conn, msg: dict, owner: str): + """Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version.""" + cur = await conn.execute( + "SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status" + " FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.id = %s FOR UPDATE OF j", + (msg["job_id"],), + ) + job = await cur.fetchone() + if job is None or str(job["attachment_id"]) != msg["attachment_id"]: + return "missing", None + if job["state"] in ("succeeded", "failed"): + return "done", job + if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]: + return "newer_version", job + if msg["attempt"] != job["attempt"]: + return "stale", job + if job["state"] == "processing" and not job["lease_expired"]: + return "busy", job + if job["state"] == "queued" and not job["due"]: + return "early", job + takeover = job["state"] == "processing" + await conn.execute( + "UPDATE attachment_jobs SET state = 'processing', lease_owner = %s," + " lease_until = clock_timestamp() + make_interval(secs => %s)," + " started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp())," + " updated_at = clock_timestamp() WHERE id = %s", + (owner, config.JOB_LEASE_SECONDS, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (job["attachment_id"],), + ) + await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"]) + return "claimed", job + + +async def heartbeat(conn, job_id, owner: str) -> bool: + cur = await conn.execute( + "UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s", + (config.JOB_LEASE_SECONDS, job_id, owner), + ) + return cur.rowcount == 1 + + +async def attachment_source(conn, attachment_id) -> Optional[dict]: + cur = await conn.execute( + "SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status" + " FROM attachments WHERE id = %s", + (attachment_id,), + ) + return await cur.fetchone() + + +async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool: + """Atomically publish the result. False when the lease was lost: someone else owns the effect now.""" + cur = await conn.execute( + "UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL," + " lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s", + (job["id"], owner), + ) + if cur.rowcount != 1: + return False + for v in variants: + await conn.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type," + " size_bytes, width, height, sha256, created_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())" + " ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING", + (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"]), + ) + # Never move a newer pipeline_version back, never touch an attachment that is already ready. + await conn.execute( + "UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()" + " WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s", + (job["pipeline_version"], job["attachment_id"], job["pipeline_version"]), + ) + await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"], + {"variants": [v["name"] for v in variants]}) + return True + + +async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str: + attempt = job["attempt"] + if transient and attempt < job["max_attempts"]: + delay = backoff_seconds(attempt) + await conn.execute( + "UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL," + " next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s," + " updated_at = clock_timestamp() WHERE id = %s", + (delay, detail[:500], error_code, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (job["attachment_id"],), + ) + await _outbox(conn, job, attempt + 1, delay=delay) + await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt, + {"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)}) + return "retry" + final_code = error_code if not transient else "retry_exhausted" + await conn.execute( + "UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp()," + " last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s", + (detail[:500], final_code, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (final_code, job["attachment_id"]), + ) + await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt, + {"error_code": final_code, "cause": error_code, "detail": detail[:200]}) + return "failed" + + +async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str: + """Returns retry | failed | lost (lease is not ours any more: the new owner decides).""" + cur = await conn.execute( + "SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE", + (job["id"], owner), + ) + row = await cur.fetchone() + if row is None: + return "lost" + return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient) + + +async def sweep_leases(conn, actor: str) -> List[str]: + """Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt.""" + cur = await conn.execute( + "SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner" + " FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()" + " ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED", + ) + outcomes = [] + for row in await cur.fetchall(): + outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired", + "lease of %s expired" % (row["lease_owner"] or "?"), True) + outcomes.append("%s:%s" % (row["id"], outcome)) + return outcomes + + +async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict: + """failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed.""" + cur = await conn.execute( + "SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at," + " a.status AS attachment_status, a.staging_key" + " FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a", + (attachment_id,), + ) + job = await cur.fetchone() + if job is None: + return {"result": "refused", "reason": "no job for attachment"} + if job["state"] != "failed" or job["attachment_status"] != "failed": + await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], + {"state": job["state"], "attachment_status": job["attachment_status"]}) + return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])} + if not job["staging_key"] or not await staging_exists(job["staging_key"]): + await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"}) + return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"} + attempt = job["attempt"] + 1 + await conn.execute( + "UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp()," + " finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s", + (attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'failed'", + (job["attachment_id"],), + ) + await _outbox(conn, job, attempt) + await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt) + return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt} + + +async def failed_jobs(conn, limit: int = 50) -> List[dict]: + cur = await conn.execute( + "SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at" + " FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s", + (limit,), + ) + return [ + {"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"], + "max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"], + "failed_at": fmt(r["updated_at"])} + for r in await cur.fetchall() + ] + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def older_than(seconds: int) -> datetime: + return utcnow() - timedelta(seconds=seconds) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..6848d3f --- /dev/null +++ b/app/main.py @@ -0,0 +1,801 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import logging +import os +import re +import resource +import signal +import tempfile +import unicodedata +import sys +import uuid +from contextlib import asynccontextmanager +from typing import List, Optional, Tuple + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from starlette.concurrency import iterate_in_threadpool, run_in_threadpool +from starlette.responses import StreamingResponse + +from app import config, faults, jobs, openbao, passwords, reconcile, repo, telemetry +from app.cursor import BadCursor, decode_cursor, encode_cursor +from app.db import Database +from app.imagecheck import inspect_image +from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable +from app.broker import Publisher +from app.outbox import Relay +from app.s3 import S3Storage +from app.sessions import SessionStore + +log = logging.getLogger("messenger") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"}) +EXEMPT_PATHS = ("/health/live", "/health/ready") +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 +ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"} +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" + +upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) +background = set() +db = Database() +sessions = SessionStore(db) +s3 = S3Storage() +publisher = Publisher() +relay = Relay(db, publisher) +state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0, + "queue": None} +VARIANTS = ("original", "thumbnail", "crop", "optimized") + + +def rss_mb() -> float: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE / 1048576 + except OSError: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak / 1048576 if sys.platform == "darwin" else peak / 1024 + + +def count_rejection(code: str) -> None: + state["rejected"][code] = state["rejected"].get(code, 0) + 1 + + +async def sweeper() -> None: + while True: + await asyncio.sleep(config.SESSION_SWEEP_SECONDS) + try: + removed = await sessions.sweep() + if removed: + log.info("sessions_expired removed=%d", removed) + except Exception: + log.warning("session_sweep_skipped: storage unavailable") + try: + async with db.connection() as conn: + removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS) + if removed: + log.info("idempotency_cleanup removed=%d", removed) + except Exception: + log.warning("idempotency_cleanup_skipped") + try: + removed = await relay.cleanup() + if removed: + log.info("outbox_cleanup removed=%d", removed) + except Exception: + log.warning("outbox_cleanup_skipped") + + +async def pending_watch() -> None: + """Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second.""" + while True: + try: + async with db.connection() as conn: + queue = await jobs.queue_stats(conn) + state["pending_jobs"] = queue["queued"] + queue["processing"] + state["queue"] = queue + except Exception: + state["queue"] = None # unknown is not zero: the gauge disappears instead of lying + await asyncio.sleep(config.PENDING_CHECK_SECONDS) + + +async def reconciler_loop() -> None: + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + while True: + try: + result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False) + if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))): + log.info("reconcile %s", result) + except Exception: + log.warning("reconcile_skipped: dependency unavailable") + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + + +async def readiness_watch() -> None: + failures = 0 + while True: + if await db.ready(): + failures = 0 + if not state["ready"]: + state["ready"] = True + log.info("ready_again: database reachable") + else: + failures += 1 + if failures >= config.READY_FAILURES and state["ready"]: + state["ready"] = False + log.warning("not_ready: database unreachable, business requests get 503") + s3_ok = await run_in_threadpool(s3.probe) + if s3_ok != state["s3_ok"]: + state["s3_ok"] = s3_ok + log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503") + await asyncio.sleep(config.READY_CHECK_SECONDS) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + previous = {} + + def on_signal(signum, frame): + if not state["draining"]: + state["draining"] = True + log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name) + handler = previous.get(signum) + if callable(handler): + handler(signum, frame) + + for sig in (signal.SIGTERM, signal.SIGINT): + previous[sig] = signal.getsignal(sig) + signal.signal(sig, on_signal) + await db.open() + telemetry.register_pool(db) + telemetry.register_queue(state) + await sessions.open() + await run_in_threadpool(s3.open) + state["s3_ok"] = await run_in_threadpool(s3.probe) + tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()), + asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())] + health_server = await asyncio.start_server(health_listener, "127.0.0.1", config.HEALTH_LISTEN_PORT) + bao = openbao.client() + if bao is not None: + # Rotation without restart: a new KV version swaps DB and S3 credentials for new connections. + bao.on_change(lambda data: db.rotate(data.get("database_url", ""))) + bao.on_change(s3.rotate) + bao.start_refresher(config.BAO_REFRESH_SECONDS) + log.info( + "started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s", + config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS, + config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES, + config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET, + ) + log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s", + config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION) + yield + for task in tasks: + task.cancel() + health_server.close() + await publisher.close() + await sessions.close() + await db.close() + log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"]) + telemetry.shutdown() + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +@app.middleware("http") +async def guard(request: Request, call_next): + rid = uuid.uuid4().hex + request.state.rid = rid + # X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request. + telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID) + exempt = request.scope["path"] in EXEMPT_PATHS + if state["draining"] and request.scope["path"] != "/health/live": + count_rejection("draining") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"}) + if not exempt and not state["ready"]: + count_rejection("not_ready") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"}) + if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS: + count_rejection("inflight") + return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"}) + state["inflight"] += 1 + try: + fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else "" + if fault: + await inject_http_fault(fault) + response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT) + except asyncio.TimeoutError: + count_rejection("request_timeout") + log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.scope["path"]) + response = error_body( + rid, config.INSTANCE_ID, 503, "unavailable", + "Запрос не удалось обработать за отведённое время", {"Retry-After": "1"}, + ) + except Exception as exc: + telemetry.record_exception(exc) + log.exception("unhandled_error request_id=%s", rid) + response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера") + finally: + state["inflight"] -= 1 + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +async def inject_http_fault(fault: str) -> None: + """Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error` + raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose + trace survival can be counted (span attribute messenger.fault).""" + telemetry.set_attributes(messenger__fault=fault[:16]) + if fault == "slow": + await asyncio.sleep(0.6) + elif fault == "error": + raise RuntimeError("injected handler failure") + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + if exc.status in (413, 429, 503): + count_rejection(exc.code) + return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers) + + +def parse_uuid(value: object) -> Optional[str]: + if not isinstance(value, str): + return None + try: + return str(uuid.UUID(value)) + except ValueError: + return None + + +def non_blank(value: object, max_len: int) -> bool: + if not isinstance(value, str) or not 1 <= len(value) <= max_len: + return False + return value.translate(ZERO_WIDTH).strip() != "" + + +async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict: + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое") + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: + raise too_large + raw = bytearray() + async for chunk in request.stream(): + raw += chunk + if len(raw) > config.MAX_BODY_BYTES: + raise too_large + try: + body = json.loads(bytes(raw).decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Некорректный JSON") from None + if not isinstance(body, dict): + raise bad("Ожидается JSON-объект") + allowed = set(required) | set(optional) + if set(body) - allowed: + raise bad("Лишние поля в запросе") + if set(required) - set(body): + raise bad("Отсутствуют обязательные поля") + return body + + +def ensure_capacity() -> None: + if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB: + raise unavailable("Достигнут лимит памяти процесса, запись недоступна") + + +def bearer_token(request: Request) -> str: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token: + raise unauthorized() + return token + + +async def actor(request: Request) -> dict: + user = await sessions.resolve(bearer_token(request)) + if user is None: + raise unauthorized() + return user + + +def basic_credentials(request: Request) -> Tuple[str, str]: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized("Basic") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized("Basic") from None + username, separator, password = decoded.partition(":") + if not separator: + raise unauthorized("Basic") + return username, password + + +async def member_chat(user: dict, chat_id: str) -> dict: + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + async with db.connection() as conn: + chat = await repo.chat_for_member(conn, cid, user["id"]) + if chat is None: + raise not_found() + return chat + + +def page_params(request: Request) -> Tuple[int, Optional[str]]: + query = request.query_params + limit = 50 + if "limit" in query: + if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100: + raise bad("limit должен быть целым 1..100") + limit = int(query["limit"]) + cursor = query.get("cursor") + if cursor is not None and cursor == "": + raise bad("Некорректный cursor") + return limit, cursor + + +def keyset(scope: str, cursor: Optional[str]): + if cursor is None: + return None + try: + return decode_cursor(scope, cursor) + except BadCursor: + raise bad("Некорректный cursor") from None + + +def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse: + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +def detect_content_type(head: bytes) -> Optional[str]: + if head[:8] == PNG_MAGIC: + return "image/png" + if head[:3] == JPEG_MAGIC: + return "image/jpeg" + return None + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."): + raise bad("filename: 1..255 символов, не только пробелы") + if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename): + raise bad("filename содержит недопустимые символы") + + +hash_slots = asyncio.Semaphore(config.PASSWORD_HASH_CONCURRENCY) + + +async def hash_slot(func, *args): + """Login DoS budget: at most PASSWORD_HASH_CONCURRENCY Argon2 computations per replica at a time (each ~19 MiB, + measured time in REPORT.md); a request that cannot get a slot quickly gets 503 instead of queueing unbounded.""" + try: + await asyncio.wait_for(hash_slots.acquire(), config.PASSWORD_HASH_QUEUE_SECONDS) + except asyncio.TimeoutError: + count_rejection("hash_busy") + raise ApiError(503, "unavailable", "Сервер перегружен проверкой паролей, повторите позже", + {"Retry-After": "1"}) from None + try: + return await run_in_threadpool(func, *args) + finally: + hash_slots.release() + + +def client_ip(request: Request) -> str: + """X-Real-IP is set by nginx. It can be trusted: only the LB identity passes the API mTLS check, so a client + cannot reach the API directly and forge it. Fallback: the TCP peer.""" + value = request.headers.get("x-real-ip", "") + return value if re.fullmatch(r"[0-9a-fA-F:.]{2,45}", value) else (request.client.host if request.client else "-") + + +def rate_limited(scope: str, retry_after: int) -> None: + count_rejection("login_rate_" + scope) + log.info("login_rate_limited scope=%s retry_after=%d", scope, retry_after) + raise ApiError(429, "rate_limited", "Слишком много попыток входа, повторите позже", + {"Retry-After": str(int(retry_after))}) + + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +async def health_listener(reader, writer) -> None: + """Loopback-only (127.0.0.1) plain HTTP for the Docker healthcheck: /health/live and /health/ready only. + It is not published and not reachable from the network, so it is no bypass of the mTLS business listener.""" + try: + line = (await asyncio.wait_for(reader.readline(), 2)).decode("latin-1") + path = line.split(" ")[1] if line.count(" ") >= 2 else "" + if path == "/health/live": + status = 200 + elif path == "/health/ready": + status = 200 if (not state["draining"] and state["s3_ok"] and await db.ready()) else 503 + else: + status = 404 + body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}' + writer.write(b"HTTP/1.1 %d X\r\nContent-Type: application/json\r\nContent-Length: %d\r\n" + b"Connection: close\r\n\r\n%s" % (status, len(body), body)) + await writer.drain() + except Exception: + pass + finally: + writer.close() + + +@app.get("/health/ready") +async def ready(): + if not await db.ready(): + raise unavailable("База данных недоступна") + if not state["s3_ok"]: + raise unavailable("Хранилище файлов недоступно") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + body = await read_json(request, ("username", "display_name", "password")) + if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + if not non_blank(body["display_name"], 100): + raise bad("display_name: 1..100 символов, не только пробелы") + password = body["password"] + if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX)) + ensure_capacity() + password_hash = await hash_slot(passwords.hash_password, password) + async with db.connection() as conn: + user = await repo.create_user(conn, body["username"], body["display_name"], password_hash) + if user is None: + raise ApiError(409, "conflict", "username уже занят") + log.info("user_created id=%s", user["id"]) + return JSONResponse(user, status_code=201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + if await request.body(): + raise bad("POST /auth/sessions не принимает тело запроса") + username, password = basic_credentials(request) + row = None + if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + async with db.connection() as conn: + row = await repo.credentials(conn, username) + # lab7: limits BEFORE the expensive Argon2 verification; counters are shared by all replicas (Valkey). + ip_key = "ratelimit:login:ip:" + client_ip(request) + user_key = "ratelimit:login:user:" + (username if USERNAME_RE.fullmatch(username) else "-invalid-") + count, retry = await sessions.hit(ip_key, config.LOGIN_RATE_WINDOW_SECONDS) + if count > config.LOGIN_RATE_LIMIT_PER_IP: + rate_limited("ip", retry) + failures, retry = await sessions.peek(user_key) + if failures >= config.LOGIN_RATE_LIMIT_PER_USER: + rate_limited("user", retry) + stored = row["password_hash"] if row else passwords.DUMMY_HASH + matches = await hash_slot(passwords.verify_password, stored, password) + if row is None or not matches: + await sessions.hit(user_key, config.LOGIN_RATE_WINDOW_SECONDS) + log.info("login_failed username_known=%s", row is not None) + raise unauthorized("Basic") + await sessions.reset(user_key) + if passwords.needs_rehash(stored): + # Old parameters (or another algorithm) -> re-hash with the current Argon2id parameters while we know the + # plaintext; a concurrent login doing the same is harmless (both hashes are valid). + new_hash = await hash_slot(passwords.hash_password, password) + async with db.connection() as conn: + await repo.update_password_hash(conn, row["id"], new_hash) + log.info("password_rehashed user=%s", row["id"]) + user = repo.public_user(row) + token, expires_at = await sessions.create(user) + log.info("session_created user=%s", user["id"]) + return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + token = bearer_token(request) + user = await sessions.resolve(token) + if user is None: + raise unauthorized() + await sessions.revoke(token) + log.info("session_revoked user=%s", user["id"]) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + return await actor(request) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await actor(request) + body = await read_json(request, ("title", "member_ids")) + if not non_blank(body["title"], 100): + raise bad("title: 1..100 символов, не только пробелы") + raw_ids = body["member_ids"] + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100: + raise bad("member_ids: массив из 1..100 UUID") + ids: List[str] = [] + for value in raw_ids: + parsed = parse_uuid(value) + if parsed is None: + raise bad("member_ids содержит некорректный UUID") + ids.append(parsed) + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user["id"] not in ids: + raise bad("Создатель должен входить в member_ids") + ensure_capacity() + + async def insert(conn): + return await repo.create_chat(conn, body["title"], ids) + + try: + chat = await db.in_transaction(insert) + except repo.UnknownMember: + raise bad("member_ids содержит неизвестного пользователя") from None + log.info("chat_created id=%s members=%d", chat["id"], len(ids)) + return JSONResponse(chat, status_code=201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await actor(request) + limit, cursor = page_params(request) + scope = "chats:" + user["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_chats(conn, user["id"], limit, after) + return page(scope, items, has_more) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + return await member_chat(await actor(request), chat_id) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user = await actor(request) + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + + idem_key = request.headers.get("idempotency-key", "") + if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key): + raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + + body = await read_json(request, ("text",), ("attachment_ids",)) + + text = body["text"] + attachment_ids_raw = body.get("attachment_ids", []) + + if not isinstance(attachment_ids_raw, list): + raise bad("attachment_ids должен быть массивом UUID") + if len(attachment_ids_raw) > 10: + raise bad("attachment_ids: максимум 10 вложений") + + attachment_ids: List[str] = [] + for value in attachment_ids_raw: + parsed = parse_uuid(value) + if parsed is None: + raise bad("attachment_ids содержит некорректный UUID") + attachment_ids.append(parsed) + + if len(set(attachment_ids)) != len(attachment_ids): + raise bad("attachment_ids содержит дубликаты") + + has_text = isinstance(text, str) and non_blank(text, 4000) + has_attachments = len(attachment_ids) > 0 + + if not has_text and not has_attachments: + raise bad("text: 1..4000 символов, не только пробелы") + + if isinstance(text, str) and len(text) > 4000: + raise bad("text: 1..4000 символов, не только пробелы") + + if not isinstance(text, str): + raise bad("text должен быть строкой") + + ensure_capacity() + + normalized = {"text": text} + if attachment_ids: + normalized["attachment_ids"] = attachment_ids + body_hash = hashlib.sha256( + json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode() + ).hexdigest() + + async def do(conn): + return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash) + + status, message = await db.in_transaction(do) + + if status == "conflict": + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса") + if status == "not_member": + raise not_found() + if status == "attachment_not_found": + raise not_found() + if status == "attachment_failed": + raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить") + return JSONResponse(message, status_code=201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + chat = await member_chat(await actor(request), chat_id) + limit, cursor = page_params(request) + scope = "messages:" + chat["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_messages(conn, chat["id"], limit, after) + return page(scope, items, has_more) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: str, request: Request): + user = await actor(request) + chat = await member_chat(user, chat_id) + + filename = request.query_params.get("filename", "") + validate_filename(filename) + + declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared_ct not in ALLOWED_CONTENT_TYPES: + raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg") + + declared_length = request.headers.get("content-length") + too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB") + if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES: + raise too_large + ensure_capacity() + # Backpressure: do not accept more work than the pipeline can hold; the client retries later. + if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS: + count_rejection("queue_full") + raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"}) + + sha = hashlib.sha256() + size = 0 + with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool: + head = b"" + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise too_large + if len(head) < 8: + head += chunk[: 8 - len(head)] + if len(head) == 8 and detect_content_type(head) is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + sha.update(chunk) + spool.write(chunk) + if size == 0: + raise bad("Пустой файл") + actual_ct = detect_content_type(head) + if actual_ct is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + if actual_ct != declared_ct: + raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла") + async with upload_slots: + await run_in_threadpool(inspect_image, spool, size, actual_ct) + + attachment_id = str(uuid.uuid4()) + staging_key = "staging/" + attachment_id + object_key = "attachments/%s/original" % attachment_id + async with db.connection() as conn: + await repo.begin_attachment( + conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), + staging_key, object_key, + ) + try: + async with upload_slots: + await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct) + except BaseException: + defer(compensate(attachment_id, staging_key)) + raise + + # Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox. + try: + att = await db.in_transaction(lambda conn: repo.queue_attachment( + conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS)) + except BaseException: + await compensate(attachment_id, staging_key) + raise + if att == repo.QUEUE_FULL: + defer(compensate(attachment_id, staging_key)) + count_rejection("queue_full") + raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"}) + if att is None: + raise unavailable("Загрузка отменена сверкой, повторите запрос") + telemetry.set_attributes(messenger__attachment_id=att["id"]) + if faults.active(filename, "api_crash_after_commit"): + faults.crash("API after COMMIT of job/outbox, before publish") + relay.notify() + log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size) + return JSONResponse(att, status_code=202) + + +def defer(coro) -> None: + task = asyncio.ensure_future(coro) + background.add(task) + task.add_done_callback(background.discard) + + +async def compensate(attachment_id: str, object_key: str) -> None: + """Best effort; whatever is left behind is collected by the reconciler after the grace period.""" + try: + async with db.connection() as conn: + await repo.abort_attachment(conn, attachment_id) + await run_in_threadpool(s3.delete, object_key) + except BaseException: + log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id) + + +async def accessible_attachment(request: Request, attachment_id: str) -> dict: + user = await actor(request) + aid = parse_uuid(attachment_id) + if aid is None: + raise bad("attachment_id должен быть UUID") + async with db.connection() as conn: + row = await repo.get_attachment(conn, aid) + if row is None: + raise not_found() + chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"]) + if chat is None: + raise not_found() + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variants = [] + if row["status"] == "ready": + async with db.connection() as conn: + variants = await repo.get_variants(conn, row["id"], row["pipeline_version"]) + return repo._attachment(row, variants) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variant = request.query_params.get("variant", "original") + if variant not in VARIANTS: + raise bad("variant: original, thumbnail, crop или optimized") + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing") + else "Обработка вложения завершилась ошибкой") + key, content_type = row["object_key"], row["content_type"] + if variant != "original": + async with db.connection() as conn: + found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant] + if not found: + raise not_found() + key, content_type = found[0]["object_key"], found[0]["content_type"] + opened = await run_in_threadpool(s3.open_stream, key) + if opened is None: + log.error("object_missing attachment=%s variant=%s", row["id"], variant) + raise unavailable("Файл временно недоступен") + chunks, length = opened + return StreamingResponse( + iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)}, + ) + + +@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def fallback(request: Request): + raise not_found() + + +# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel +# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics. +telemetry.setup("messenger-api", app) diff --git a/app/openbao.py b/app/openbao.py new file mode 100644 index 0000000..46e92ae --- /dev/null +++ b/app/openbao.py @@ -0,0 +1,161 @@ +"""OpenBao client for the services (lab7): cert auth -> short-lived token -> read ONE scoped KV v2 secret. + +Bootstrap trust: the only material a service gets from the operator is its TLS client certificate (Docker secret). +It is also its identity for RabbitMQ (EXTERNAL) and LB->API mTLS is separate. There is no "secret to get secrets" +in the image or in .env: no token, no AppRole SecretID. + + login POST /v1/auth/cert/login (client certificate, role = BAO_AUTH_ROLE) -> token (TTL 15 min, max 1 h) + read GET /v1/messenger/data/ (policy messenger-: read own path only) + refresh every BAO_REFRESH_SECONDS: renew-self (or re-login near max TTL), re-read metadata; a new KV version + (rotation) is delivered to registered callbacks without a restart + sealed / unreachable OpenBao: the process keeps the secrets it already holds (bounded: existing DB/S3/broker + sessions continue); a NEW process waits up to BAO_STARTUP_TIMEOUT_SECONDS and then exits non-zero. +Secret values are never logged; only paths, versions, TTLs and error classes. +""" +import json +import logging +import os +import ssl +import threading +import time +import urllib.error +import urllib.request +from typing import Callable, Dict, List, Optional + +log = logging.getLogger("messenger.openbao") + + +class OpenBaoError(Exception): + pass + + +class OpenBao: + def __init__(self) -> None: + self.addr = os.environ["BAO_ADDR"].rstrip("/") + self.role = os.environ["BAO_AUTH_ROLE"] + self.path = os.environ["BAO_KV_PATH"] + self.mount = os.environ.get("BAO_KV_MOUNT", "messenger") + self.ctx = ssl.create_default_context(cafile=os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt")) + self.ctx.load_cert_chain(os.environ.get("TLS_CLIENT_CERT_FILE", "/run/secrets/client.crt"), + os.environ.get("TLS_CLIENT_KEY_FILE", "/run/secrets/client.key")) + self.token: Optional[str] = None + self.token_expires = 0.0 + self.data: Dict[str, str] = {} + self.version = 0 + self.callbacks: List[Callable[[Dict[str, str]], None]] = [] + self.state = {"available": False, "last_ok": 0.0, "errors": 0, "logins": 0, "renewals": 0} + self.lock = threading.Lock() + + # ------------------------------------------------------------------ HTTP + def _call(self, method: str, path: str, body: Optional[dict] = None, token: Optional[str] = None) -> dict: + req = urllib.request.Request(self.addr + path, method=method, + data=json.dumps(body).encode() if body is not None else None) + req.add_header("Content-Type", "application/json") + if token: + req.add_header("X-Vault-Token", token) + try: + with urllib.request.urlopen(req, context=self.ctx, timeout=5) as response: + raw = response.read() + except urllib.error.HTTPError as exc: + # 503 = sealed / standby, 403 = denied or expired token, 400 = cert not accepted for the role + raise OpenBaoError("HTTP %d on %s" % (exc.code, path.split("?")[0])) from None + except (urllib.error.URLError, OSError) as exc: + raise OpenBaoError("%s on %s" % (type(getattr(exc, "reason", exc)).__name__, path)) from None + return json.loads(raw) if raw else {} + + def login(self) -> None: + auth = self._call("POST", "/v1/auth/cert/login", {"name": self.role})["auth"] + now = time.time() + self.token = auth["client_token"] + self.token_expires = now + auth["lease_duration"] + self.state["logins"] += 1 + log.info("openbao_login role=%s policies=%s ttl=%ds", self.role, ",".join(auth.get("policies", [])), + auth["lease_duration"]) + + def _ensure_token(self, interval: float) -> None: + now = time.time() + if self.token is None or now > self.token_expires - 1: + self.login() + return + if self.token_expires - now < 2 * interval + 5: + try: + auth = self._call("POST", "/v1/auth/token/renew-self", {}, self.token)["auth"] + self.token_expires = time.time() + auth["lease_duration"] + self.state["renewals"] += 1 + if auth["lease_duration"] < 2 * interval + 5: # reached max TTL: authenticate again + self.login() + except OpenBaoError: + self.login() + + def _read(self) -> None: + body = self._call("GET", "/v1/%s/data/%s" % (self.mount, self.path), token=self.token)["data"] + version = int(body["metadata"]["version"]) + changed = version != self.version + with self.lock: + self.data, self.version = dict(body["data"]), version + self.state.update(available=True, last_ok=time.time()) + if changed: + log.info("openbao_secret_loaded path=%s/%s version=%d keys=%s", self.mount, self.path, version, + ",".join(sorted(self.data))) + for callback in list(self.callbacks): + try: + callback(dict(self.data)) + except Exception: + log.exception("openbao_rotation_callback_failed") + + # ------------------------------------------------------------------ lifecycle + def load_blocking(self, timeout: float) -> Dict[str, str]: + deadline = time.monotonic() + timeout + delay = 1.0 + while True: + try: + self.login() + self._read() + return dict(self.data) + except (OpenBaoError, KeyError, ValueError) as exc: + self.state["errors"] += 1 + if time.monotonic() > deadline: + log.error("openbao_unavailable_at_start role=%s error=%s: giving up", self.role, exc) + raise SystemExit(78) + log.warning("openbao_not_ready role=%s error=%s (sealed or down?), retry in %.0fs", self.role, exc, delay) + time.sleep(delay) + delay = min(delay * 2, 10.0) + + def on_change(self, callback: Callable[[Dict[str, str]], None]) -> None: + self.callbacks.append(callback) + + def start_refresher(self, interval: float) -> None: + def loop(): + while True: + time.sleep(interval) + try: + self._ensure_token(interval) + meta = self._call("GET", "/v1/%s/metadata/%s" % (self.mount, self.path), token=self.token) + if int(meta["data"]["current_version"]) != self.version: + self._read() + else: + self.state.update(available=True, last_ok=time.time()) + except (OpenBaoError, KeyError, ValueError) as exc: + self.state["errors"] += 1 + if self.state["available"]: + log.warning("openbao_unavailable role=%s error=%s: keeping cached secret version %d", + self.role, exc, self.version) + self.state["available"] = False + if "403" in str(exc): + self.token = None # expired/revoked token: log in again next round + + threading.Thread(target=loop, name="openbao-refresh", daemon=True).start() + + +_client: Optional[OpenBao] = None + + +def client() -> Optional[OpenBao]: + return _client + + +def load(timeout: float) -> Dict[str, str]: + """Called once by app.config at import time when BAO_ADDR is set.""" + global _client + _client = OpenBao() + return _client.load_blocking(timeout) diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 0000000..6e7d854 --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,100 @@ +"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work). + +Rows are locked while they are published; `published_at` is set in the same transaction only after the +broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are +expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish +leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS. +""" +import asyncio +import logging + +from opentelemetry.trace import SpanKind + +from app import config, telemetry + +log = logging.getLogger("messenger.outbox") + + +class Relay: + def __init__(self, db, publisher) -> None: + self.db = db + self.publisher = publisher + self.wake = asyncio.Event() + self.broker_ok = False + self.published = 0 + + def notify(self) -> None: + self.wake.set() + + async def run_once(self) -> int: + sent, failure = 0, None + async with self.db.connection() as conn: + async with conn.transaction(): + cur = await conn.execute( + "SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()" + " ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED", + (config.OUTBOX_BATCH,), + ) + for row in await cur.fetchall(): + try: + await self.publish(row["payload"]) + except Exception as exc: + # Keep what was already confirmed: commit those rows, record the error, stop the batch. + await conn.execute( + "UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s", + (type(exc).__name__, row["id"]), + ) + failure = exc + break + await conn.execute( + "UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1" + " WHERE id = %s", + (row["id"],), + ) + sent += 1 + self.published += sent + if failure is not None: + raise failure + return sent + + async def publish(self, payload: dict) -> None: + """PRODUCER span, child of the context stored with the job (upload request or failed attempt); + the message then carries the producer's own traceparent, so the consumer becomes its child.""" + attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "publish", + "messaging.destination.name": config.BROKER_EXCHANGE, "messaging.message.id": payload["job_id"], + "messenger.job.attempt": payload["attempt"], "messenger.attachment_id": payload["attachment_id"]} + with telemetry.span(config.BROKER_QUEUE + " publish", kind=SpanKind.PRODUCER, + context=telemetry.extract(payload), attributes=attrs): + body = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")} + body.update(telemetry.inject()) + await self.publisher.publish(body) + + async def loop(self) -> None: + delay = config.OUTBOX_POLL_SECONDS + while True: + try: + await asyncio.wait_for(self.wake.wait(), delay) + except asyncio.TimeoutError: + pass + self.wake.clear() + try: + while await self.run_once() == config.OUTBOX_BATCH: + pass + if not self.broker_ok: + log.info("outbox_relay_resumed: broker reachable") + self.broker_ok = True + delay = config.OUTBOX_POLL_SECONDS + except Exception as exc: + if self.broker_ok: + log.warning("outbox_relay_paused: %s; jobs stay in outbox", type(exc).__name__) + self.broker_ok = False + await self.publisher.close() + delay = min(10.0, max(delay * 2, 1.0)) + + async def cleanup(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute( + "DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)", + (config.OUTBOX_RETENTION_SECONDS,), + ) + return cur.rowcount diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 0000000..2f3d6ae --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,29 @@ +"""Argon2id password storage (lab7). Parameters come from the environment and were measured on the stand +(student-tests/argon2_bench.py, REPORT.md); the PHC string stores salt + parameters, so old hashes stay valid and +are upgraded on the next successful login (needs_rehash).""" +from argon2 import PasswordHasher, Type +from argon2.exceptions import Argon2Error, InvalidHashError, VerifyMismatchError + +from app import config + +HASHER = PasswordHasher(time_cost=config.ARGON2_ITERATIONS, memory_cost=config.ARGON2_MEMORY_KIB, + parallelism=config.ARGON2_PARALLELISM, hash_len=32, salt_len=16, type=Type.ID) +DUMMY_HASH = HASHER.hash("there-is-no-such-user-password") + + +def hash_password(password: str) -> str: + return HASHER.hash(password) + + +def verify_password(stored: str, password: str) -> bool: + try: + return HASHER.verify(stored, password) + except (VerifyMismatchError, Argon2Error, InvalidHashError, ValueError): + return False + + +def needs_rehash(stored: str) -> bool: + try: + return HASHER.check_needs_rehash(stored) + except (InvalidHashError, ValueError): + return True diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 0000000..60a138f --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,155 @@ +"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout). + +Key layout (all keys are generated by the server): + staging/ written by the API before 202, read by the worker + attachments//original server-side copy made by the worker + attachments//v/.* variants of pipeline version N +Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row. + +Cases (all gated by a grace period so in-flight work is never touched): + stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both + orphan_object object without any row/variant row, older than grace -> delete object + staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete + staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete + (after that a replay is refused; the attachment stays failed and visible) + missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing +Queued/processing work is never touched: its staging object is the input of a live job. +""" +import logging +import time +from datetime import datetime, timedelta, timezone + +from starlette.concurrency import run_in_threadpool + +from app import config + +log = logging.getLogger("messenger.reconcile") + +LOCK_ID = 2026_0104 +PREFIXES = ("staging/", "attachments/") +BATCH = 500 + + +async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict: + started = time.monotonic() + result = { + "dry_run": dry_run, "grace_seconds": grace_seconds, + "stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0, + "deleted_objects": 0, "removed_rows": 0, "marked_failed": 0, + "objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0, + } + async with db.connection() as conn: + cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,)) + if not (await cur.fetchone())["got"]: + return {"skipped": "another reconciler is running"} + try: + now = datetime.now(timezone.utc) + cutoff = now - timedelta(seconds=grace_seconds) + await _stale_uploading(conn, s3, cutoff, dry_run, result) + await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result) + await _orphan_objects(conn, s3, cutoff, dry_run, result) + await _missing_objects(conn, s3, cutoff, dry_run, result) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + result["seconds"] = round(time.monotonic() - started, 3) + return result + + +async def _stale_uploading(conn, s3, cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, coalesce(staging_key, object_key) AS key FROM attachments" + " WHERE status = 'uploading' AND created_at < %s", + (cutoff,), + ) + for row in await cur.fetchall(): + modified = await run_in_threadpool(s3.head, row["key"]) + if modified is not None and modified >= cutoff: + result["protected_fresh_objects"] += 1 + continue + result["stale_uploading"] += 1 + if dry_run: + continue + if modified is not None: + await run_in_threadpool(s3.delete, row["key"]) + result["deleted_objects"] += 1 + done = await conn.execute( + "DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff), + ) + result["removed_rows"] += done.rowcount + + +async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL" + " AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))", + (retention_cutoff,), + ) + for row in await cur.fetchall(): + if await run_in_threadpool(s3.head, row["staging_key"]) is None: + continue + result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1 + if dry_run: + continue + again = await conn.execute( + "SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))", + (row["id"], retention_cutoff), + ) + if await again.fetchone() is not None: + await run_in_threadpool(s3.delete, row["staging_key"]) + result["deleted_objects"] += 1 + + +async def _known_keys(conn, keys): + cur = await conn.execute( + "SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)" + " UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)" + " UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)", + (keys, keys, keys), + ) + return {row["k"] for row in await cur.fetchall()} + + +async def _orphan_objects(conn, s3, cutoff, dry_run, result): + for prefix in PREFIXES: + pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix))) + while True: + page = await run_in_threadpool(next, pages, None) + if page is None: + break + result["objects_scanned"] += len(page) + keys = [o["Key"] for o in page] + if not keys: + continue + known = await _known_keys(conn, keys) + for obj in page: + if obj["Key"] in known or obj["LastModified"] >= cutoff: + continue + result["orphan_objects"] += 1 + if not dry_run and not await _known_keys(conn, [obj["Key"]]): + await run_in_threadpool(s3.delete, obj["Key"]) + result["deleted_objects"] += 1 + + +async def _missing_objects(conn, s3, cutoff, dry_run, result): + last = None + while True: + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s" + " AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s", + (cutoff, last, last, BATCH), + ) + rows = await cur.fetchall() + if not rows: + return + for row in rows: + result["rows_scanned"] += 1 + if await run_in_threadpool(s3.exists, row["object_key"]): + continue + result["missing_objects"] += 1 + if not dry_run: + done = await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'object_missing'," + " updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],), + ) + result["marked_failed"] += done.rowcount + last = str(rows[-1]["id"]) diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 0000000..b425043 --- /dev/null +++ b/app/repo.py @@ -0,0 +1,342 @@ +import hashlib +import json +from typing import List, Optional, Tuple + +import psycopg + +from app import config, jobs +from app.cursor import fmt + +MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" +CHAT_QUERY = ( + "SELECT c.id, c.title, c.created_at," + " ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids" +) + + +class UnknownMember(Exception): + pass + + +def decoded(value) -> str: + return bytes(value).decode("utf-8") + + +def public_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": decoded(row["display_name"]), + "created_at": fmt(row["created_at"]), + } + + +def _chat(row) -> dict: + return { + "id": str(row["id"]), + "title": decoded(row["title"]), + "member_ids": [str(value) for value in row["member_ids"]], + "created_at": fmt(row["created_at"]), + } + + +def _message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": decoded(row["text"]), + "created_at": fmt(row["created_at"]), + } + + +def _message_with_attachments(row, attachment_ids: List[str]) -> dict: + msg = _message(row) + msg["attachment_ids"] = attachment_ids + return msg + + +VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2} + + +def _variant(row) -> dict: + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + + +def _attachment(row, variants=None) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [], + "error_code": row["error_code"], + "created_at": fmt(row["created_at"]), + "updated_at": fmt(row["updated_at"]), + } + + +async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]: + try: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())" + " RETURNING id, username, display_name, created_at", + (username, display_name.encode("utf-8"), password_hash), + ) + except psycopg.errors.UniqueViolation: + return None + return public_user(await cur.fetchone()) + + +async def update_password_hash(conn, user_id, password_hash: str) -> None: + await conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (password_hash, user_id)) + + +async def credentials(conn, username: str): + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,), + ) + return await cur.fetchone() + + +async def create_chat(conn, title: str, member_ids: List[str]) -> dict: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_at)" + " VALUES (gen_random_uuid(), %s, clock_timestamp())" + " RETURNING id, title, created_at", + (title.encode("utf-8"),), + ) + chat = await cur.fetchone() + cur = await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])", + (chat["id"], member_ids), + ) + if cur.rowcount != len(member_ids): + raise UnknownMember() + chat["member_ids"] = list(member_ids) + return _chat(chat) + + +async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]: + cur = await conn.execute( + CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id), + ) + row = await cur.fetchone() + return _chat(row) if row else None + + +async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + CHAT_QUERY + + " FROM chats c JOIN chat_members m ON m.chat_id = c.id" + " WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))" + " ORDER BY c.created_at, c.id LIMIT %s", + (user_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + return [_chat(row) for row in rows[:limit]], len(rows) > limit + + +async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]: + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id), + ) + row = await cur.fetchone() + return _message(row) if row else None + + +async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]: + cur = await conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ) + rows = await cur.fetchall() + return [str(row["attachment_id"]) for row in rows] + + +async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None: + for pos, aid in enumerate(attachment_ids): + await conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message_id, aid, pos), + ) + + +async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + "SELECT " + MESSAGE_COLUMNS + " FROM messages" + " WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))" + " ORDER BY created_at, id LIMIT %s", + (chat_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + messages = [] + for row in rows[:limit]: + aids = await _fetch_message_attachment_ids(conn, row["id"]) + messages.append(_message_with_attachments(row, aids)) + return messages, len(rows) > limit + + +def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int: + raw = "%s:%s:%s" % (user_id, chat_id, key) + return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16) + + +async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str, + attachment_ids: List[str], idem_key: str, body_hash: str): + lock_id = _idem_lock_id(sender_id, chat_id, idem_key) + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ) + if await cur.fetchone() is None: + return "not_member", None + + cur = await conn.execute( + "SELECT request_body_hash, response_json FROM idempotency_keys" + " WHERE user_id = %s AND chat_id = %s AND key = %s", + (sender_id, chat_id, idem_key), + ) + existing = await cur.fetchone() + + if existing: + if existing["request_body_hash"] != body_hash: + return "conflict", None + return "replay", json.loads(existing["response_json"]) + + if attachment_ids: + cur = await conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'", + (attachment_ids,), + ) + att_rows = await cur.fetchall() + found_ids = {str(row["id"]) for row in att_rows} + for aid in attachment_ids: + if aid not in found_ids: + return "attachment_not_found", None + for row in att_rows: + if str(row["chat_id"]) != chat_id: + return "attachment_not_found", None + if row["status"] == "failed": + return "attachment_failed", None + + text_bytes = text.encode("utf-8") if text else b"" + + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text_bytes, chat_id, sender_id), + ) + row = await cur.fetchone() + if row is None: + return "not_member", None + + if attachment_ids: + await _link_attachments(conn, row["id"], chat_id, attachment_ids) + + message = _message_with_attachments(row, list(attachment_ids)) + + await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)" + " VALUES (%s, %s, %s, %s, %s, clock_timestamp())", + (sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)), + ) + + return "created", message + + +async def cleanup_idempotency(conn, ttl_seconds: int) -> int: + cur = await conn.execute( + "DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)", + (ttl_seconds,), + ) + return cur.rowcount + + +async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str, + content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None: + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key," + " staging_key, status, error_code, pipeline_version, created_at, updated_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())", + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, + config.PIPELINE_VERSION), + ) + + +ATTACHMENT_COLUMNS = ( + "id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status," + " error_code, pipeline_version, created_at, updated_at" +) + + +QUEUE_FULL = "queue_full" +ADMISSION_LOCK = 2026_0105 + + +async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0): + """uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff). + + With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock, + so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight. + """ + if max_pending: + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,)) + if await jobs.pending_count(conn) >= max_pending: + return QUEUE_FULL + cur = await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS, + (attachment_id,), + ) + row = await cur.fetchone() + if row is None: + return None + await jobs.enqueue(conn, attachment_id, actor) + return _attachment(row) + + +async def abort_attachment(conn, attachment_id: str) -> None: + await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,)) + + +async def get_attachment(conn, attachment_id: str): + cur = await conn.execute( + "SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'", + (attachment_id,), + ) + return await cur.fetchone() + + +async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]: + cur = await conn.execute( + "SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants" + " WHERE attachment_id = %s AND pipeline_version = %s", + (attachment_id, pipeline_version), + ) + rows = await cur.fetchall() + return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99)) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 0000000..57ded13 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,132 @@ +import logging +from typing import Iterator, Optional, Tuple + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.errors import unavailable + +log = logging.getLogger("messenger.s3") + +MISSING = ("NoSuchKey", "404", "NotFound") + + +class S3Storage: + def __init__(self) -> None: + self.client = None + self.probe_client = None + + def open(self, access_key: str = "", secret_key: str = "") -> None: + self.access_key = access_key or config.S3_ACCESS_KEY_ID + self.secret_key = secret_key or config.S3_SECRET_ACCESS_KEY + self.probe_client = self._client(1.0, 1.5) + self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT) + + def rotate(self, secrets: dict) -> None: + """New S3 credentials from OpenBao: swap the clients (boto3 clients are immutable).""" + key, secret = secrets.get("s3_access_key_id"), secrets.get("s3_secret_access_key") + if key and secret and (key, secret) != (self.access_key, self.secret_key): + self.open(key, secret) + log.info("s3_credentials_rotated access_key=%s", key) + + def _client(self, connect: float, read: float): + return boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + verify=config.TLS_CA_FILE, # lab7: HTTPS to MinIO, server certificate verified against the lab CA + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect, + read_timeout=read, + retries={"max_attempts": 0}, + ), + ) + + def probe(self) -> bool: + try: + self.probe_client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False + + def put(self, key: str, fileobj, size: int, content_type: str) -> None: + try: + fileobj.seek(0) + self.client.put_object( + Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type, + ) + except (BotoCoreError, ClientError) as exc: + log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + + def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]: + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + except BotoCoreError as exc: + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + body = response["Body"] + return body.iter_chunks(65536), int(response["ContentLength"]) + + def head(self, key: str): + """LastModified of the object, or None when it does not exist.""" + try: + return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"] + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + raise + + def exists(self, key: str) -> bool: + return self.head(key) is not None + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=config.S3_BUCKET, Key=key) + + def list_pages(self, prefix: str): + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}): + yield page.get("Contents", []) + + # ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient) + + def put_file(self, key: str, path: str, content_type: str) -> None: + with open(path, "rb") as handle: + self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type) + + def download(self, key: str, path: str) -> bool: + """False when the object does not exist.""" + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return False + raise + with open(path, "wb") as handle: + for chunk in response["Body"].iter_chunks(1 << 20): + handle.write(chunk) + return True + + def copy(self, source_key: str, target_key: str) -> None: + """Server-side copy: the original bytes never pass through the worker again.""" + self.client.copy_object( + Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key}, + ) + + def ready(self) -> bool: + try: + self.client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False diff --git a/app/serve.py b/app/serve.py new file mode 100644 index 0000000..805557c --- /dev/null +++ b/app/serve.py @@ -0,0 +1,95 @@ +"""API process entrypoint (lab7): uvicorn behind MANDATORY mutual TLS with an identity allowlist. + +TLS layer (ssl.SSLContext, CERT_REQUIRED): the client must present a certificate chained to the lab root CA. +Identity layer (this module, right after the handshake, for every connection including resumed TLS sessions): + * DNS SAN must be in ALLOWED_CLIENT_IDENTITIES (only the LB: lb.client.messenger); + * the certificate serial must not be in REVOKED_SERIALS_FILE (re-read when it changes, no restart needed). +A certificate from the same CA with another identity (worker, api, intruder) is closed before any HTTP byte is read. +There is no plaintext listener on the network; the Docker healthcheck uses 127.0.0.1:8081 (app.main). +""" +import logging +import os +import ssl +import time + +import uvicorn +from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol + +from app import config, telemetry + +log = logging.getLogger("messenger.mtls") + + +class Revocations: + def __init__(self, path: str) -> None: + self.path, self.mtime, self.checked, self.serials = path, None, 0.0, frozenset() + + def current(self) -> frozenset: + now = time.monotonic() + if self.path and now - self.checked > 1.0: + self.checked = now + try: + mtime = os.stat(self.path).st_mtime + if mtime != self.mtime: + with open(self.path, encoding="utf-8") as handle: + self.serials = frozenset(line.strip().upper() for line in handle if line.strip()) + self.mtime = mtime + log.info("mtls_revocations_loaded count=%d", len(self.serials)) + except OSError: + pass # keep the last known list + return self.serials + + +REVOKED = Revocations(config.REVOKED_SERIALS_FILE) +rejected = telemetry.meter.create_counter("messenger.mtls.rejected", unit="{connection}", + description="API mTLS connections closed by the identity check, by reason") + + +def identity_check(cert: dict) -> tuple: + """(allowed, reason, identity, serial) for a validated peer certificate dict (ssl.getpeercert()).""" + sans = [value for kind, value in cert.get("subjectAltName", ()) if kind == "DNS"] + serial = str(cert.get("serialNumber", "")).upper() + identity = sans[0] if sans else "-" + if serial in REVOKED.current(): + return False, "revoked", identity, serial + if not config.ALLOWED_CLIENT_IDENTITIES.intersection(sans): + return False, "identity_not_allowed", identity, serial + return True, "ok", identity, serial + + +class MutualTLSProtocol(HttpToolsProtocol): + def connection_made(self, transport) -> None: + super().connection_made(transport) + ssl_object = transport.get_extra_info("ssl_object") + cert = ssl_object.getpeercert() if ssl_object is not None else None + allowed, reason, identity, serial = identity_check(cert or {}) if cert else (False, "no_certificate", "-", "-") + if not allowed: + rejected.add(1, {"reason": reason}) + log.warning("mtls_client_rejected reason=%s identity=%s serial=%s", reason, identity, serial) + transport.abort() + + +def ssl_context() -> ssl.SSLContext: + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=config.TLS_CA_FILE) + ctx.load_cert_chain(config.TLS_SERVER_CERT_FILE, config.TLS_SERVER_KEY_FILE) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + return ctx + + +class Config(uvicorn.Config): + def load(self) -> None: + super().load() + self.ssl = ssl_context() # replace uvicorn's context: CA + CERT_REQUIRED + TLS >= 1.2 + + +def main() -> None: + cfg = Config("app.main:app", host=os.environ.get("APP_HOST", "0.0.0.0"), port=int(os.environ.get("APP_PORT", "8080")), + http=MutualTLSProtocol, access_log=False, timeout_graceful_shutdown=int(config.GRACEFUL_TIMEOUT), + ssl_certfile=config.TLS_SERVER_CERT_FILE, ssl_keyfile=config.TLS_SERVER_KEY_FILE, + ssl_ca_certs=config.TLS_CA_FILE, ssl_cert_reqs=ssl.CERT_REQUIRED) + uvicorn.Server(cfg).run() + + +if __name__ == "__main__": + main() diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 0000000..9404c14 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,211 @@ +import hashlib +import json +import logging +import secrets +import time +from datetime import datetime, timedelta, timezone +from typing import Optional + +import valkey.asyncio as valkey + +from app import config, repo +from app.cursor import fmt +from app.errors import unavailable + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionStore: + def __init__(self, db) -> None: + self.db = db + self.client = None + self.stats = {"hits": 0, "misses": 0, "errors": 0} + self._local = {} + + async def open(self) -> None: + if not config.CACHE_URL: + log.info("session cache disabled: CACHE_URL is empty") + return + # lab7: rediss:// = TLS to Valkey, server verified against the lab CA (hostname "valkey"); ACL user `api`. + self.client = valkey.Valkey.from_url( + config.CACHE_URL, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ssl_ca_certs=config.TLS_CA_FILE, + ssl_cert_reqs="required", + ssl_check_hostname=True, + ) + + async def close(self) -> None: + if self.client is not None: + await self.client.aclose() + self.client = None + + @staticmethod + def digest(token: str) -> bytes: + return hashlib.sha256(token.encode("utf-8")).digest() + + @staticmethod + def key(digest: bytes) -> str: + return "session:" + digest.hex() + + async def create(self, user: dict) -> tuple: + token = secrets.token_urlsafe(32) + digest = self.digest(token) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + async with self.db.connection() as conn: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at)" + " VALUES (%s, %s, clock_timestamp(), %s)", + (digest, user["id"], expires_at), + ) + await self._cache_put(digest, user, expires_at) + return token, fmt(expires_at) + + async def resolve(self, token: str) -> Optional[dict]: + digest = self.digest(token) + cached = await self._cache_get(digest) + if cached == REVOKED: + self.stats["hits"] += 1 + return None + if cached is not None: + self.stats["hits"] += 1 + return cached + self.stats["misses"] += 1 + async with self.db.connection() as conn: + cur = await conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token_hash = %s AND s.expires_at > now()", + (digest,), + ) + row = await cur.fetchone() + if row is None: + return None + user = repo.public_user(row) + await self._cache_put(digest, user, row["expires_at"]) + return user + + async def revoke(self, token: str) -> None: + digest = self.digest(token) + await self._cache_revoke(digest) + try: + async with self.db.connection() as conn: + await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,)) + except Exception: + await self._cache_release(digest) + raise + + async def sweep(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + return cur.rowcount + + def _cache_ttl(self, expires_at: datetime) -> int: + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + return min(config.SESSION_CACHE_TTL_SECONDS, remaining) + + async def _cache_get(self, digest: bytes) -> Optional[dict]: + if self.client is None: + return None + try: + raw = await self.client.get(self.key(digest)) + except Exception: + self._cache_failed("get") + return None + if not raw: + return None + if raw == REVOKED: + return REVOKED + try: + return json.loads(raw) + except ValueError: + return None + + async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None: + if self.client is None: + return + ttl = self._cache_ttl(expires_at) + if ttl <= 0: + return + payload = json.dumps(user) + try: + await self.client.set(self.key(digest), payload, ex=ttl, nx=True) + except Exception: + self._cache_failed("set") + + async def _cache_release(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.delete(self.key(digest)) + except Exception: + self._cache_failed("release") + + async def _cache_revoke(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS) + except Exception: + self._cache_failed("revoke") + raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None + + def _cache_failed(self, operation: str) -> None: + self.stats["errors"] += 1 + if self.stats["errors"] % 50 == 1: + log.warning("session_cache_failed %s, falling back to postgres (errors=%d)", operation, self.stats["errors"]) + + # ------------------------------------------------------------------ lab7: shared login rate limit + async def hit(self, key: str, window: int) -> tuple: + """INCR a fixed-window counter shared by all replicas. Returns (count, retry_after_seconds). + Valkey unavailable -> per-process fallback window (limits stay on, but are not shared).""" + if self.client is not None: + try: + count = await self.client.incr(key) + if count == 1: + await self.client.expire(key, window) + ttl = window + else: + ttl = await self.client.ttl(key) + if ttl < 0: # lost EXPIRE (crash between INCR and EXPIRE): repair + await self.client.expire(key, window) + ttl = window + return count, max(1, ttl) + except Exception: + self._cache_failed("ratelimit") + return self._local_hit(key, window) + + async def peek(self, key: str) -> tuple: + if self.client is not None: + try: + raw = await self.client.get(key) + ttl = await self.client.ttl(key) if raw else 0 + return int(raw or 0), max(1, ttl) + except Exception: + self._cache_failed("ratelimit") + entry = self._local.get(key) + if entry and entry[1] > time.monotonic(): + return entry[0], max(1, int(entry[1] - time.monotonic())) + return 0, 1 + + async def reset(self, key: str) -> None: + self._local.pop(key, None) + if self.client is not None: + try: + await self.client.delete(key) + except Exception: + self._cache_failed("ratelimit") + + def _local_hit(self, key: str, window: int) -> tuple: + now = time.monotonic() + count, until = self._local.get(key, (0, now + window)) + if until <= now: + count, until = 0, now + window + self._local[key] = (count + 1, until) + if len(self._local) > 10000: # bounded memory under a username-spraying attack + self._local = {k: v for k, v in self._local.items() if v[1] > now} + return count + 1, max(1, int(until - now)) diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..6922c0d --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,393 @@ +"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector. + +Everything is configured from the standard OTEL_* environment variables plus a few of our own: + + OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline) + OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision) + OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs) + OTEL_METRIC_EXPORT_INTERVAL metric push period, ms + OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request + +Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed +URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job +type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels. +""" +import json +import logging +import os +import re +import resource +import sys +import time +from contextlib import contextmanager +from typing import Optional + +from opentelemetry import context as otel_context +from opentelemetry import metrics, propagate, trace +from opentelemetry.trace import SpanKind, Status, StatusCode + +from app import config + +ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true" +SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0") +TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") +QUEUE = config.BROKER_QUEUE +JOB_TYPE = "attachment.process.v1" + +# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md. +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30) +JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120) +READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600) + +tracer = trace.get_tracer("messenger") +meter = metrics.get_meter("messenger") +_state = {"service": "messenger", "providers": []} + + +# ---------------------------------------------------------------------------------------------- logging + +class EventFilter(logging.Filter): + """Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word.""" + + EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)") + + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "event", None): + match = self.EVENT_RE.match(str(record.msg)) + record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log" + return True + + +class JsonFormatter(logging.Formatter): + """One JSON object per line on stdout, with trace_id/span_id of the active span (container logs).""" + + def format(self, record: logging.LogRecord) -> str: + out = { + "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs, + "level": record.levelname, + "service": _state["service"], + "event": getattr(record, "event", ""), + "logger": record.name, + "msg": record.getMessage(), + } + ctx = trace.get_current_span().get_span_context() + if ctx.is_valid: + out["trace_id"] = format(ctx.trace_id, "032x") + out["span_id"] = format(ctx.span_id, "016x") + if record.exc_info: + out["exc"] = self.formatException(record.exc_info) + return json.dumps(out, ensure_ascii=False) + + +def _setup_logging(logger_provider=None) -> None: + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + root.setLevel(logging.INFO) + stdout = logging.StreamHandler(sys.stdout) + stdout.setFormatter(JsonFormatter()) + stdout.addFilter(EventFilter()) + root.addHandler(stdout) + # The exporter's own warnings (backend down) must not be fed back into the exporter. + sdk_log = logging.getLogger("opentelemetry") + sdk_log.propagate = False + sdk_log.addHandler(stdout) + for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"): + logging.getLogger(noisy).setLevel(logging.WARNING) + if logger_provider is not None: + from opentelemetry.sdk._logs import LoggingHandler + + otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otlp.addFilter(EventFilter()) + root.addHandler(otlp) + + +# ---------------------------------------------------------------------------------------------- sampling + +def _build_sampler(): + """ParentBased(root=KindFilter(TraceIdRatio)). + + Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling, + lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces. + Children always follow the parent's decision, so an upload and its worker job are sampled together. + """ + from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult, + TraceIdRatioBased) + + ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0")) + decisions = meter.create_counter( + "messenger.telemetry.sampler.decisions", unit="{span}", + description="Head sampler decisions for spans (sample/drop) and the rule that decided") + + class KindFilter(Sampler): + def __init__(self, delegate): + self.delegate = delegate + + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER): + decisions.add(1, {"decision": "drop", "rule": "root_kind"}) + return SamplingResult(Decision.DROP) + result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links, + trace_state) + decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"}) + return result + + def get_description(self): + return "KindFilter{%s}" % self.delegate.get_description() + + class CountingParentBased(ParentBased): + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state) + parent = trace.get_current_span(parent_context).get_span_context() + if parent.is_valid: + decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", + "rule": "parent"}) + return result + + return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio))) + + +class _CountingExporter: + """Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side).""" + + def __init__(self, delegate): + from opentelemetry.sdk.trace.export import SpanExportResult + + self.delegate = delegate + self.ok = SpanExportResult.SUCCESS + self.counter = meter.create_counter( + "messenger.telemetry.spans.exported", unit="{span}", + description="Spans handed to the OTLP exporter by outcome (success/failure)") + + def export(self, spans): + result = self.delegate.export(spans) + self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"}) + return result + + def shutdown(self): + return self.delegate.shutdown() + + def force_flush(self, timeout_millis: int = 30000): + return self.delegate.force_flush(timeout_millis) + + +# ---------------------------------------------------------------------------------------------- setup + +def setup(service_name: str, fastapi_app=None) -> None: + _state["service"] = service_name + if not ENABLED: + _setup_logging(None) + logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true") + return + + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab). + res = Resource.create({ + "service.name": service_name, + "service.namespace": "messenger", + "service.version": SERVICE_VERSION, + "service.instance.id": config.INSTANCE_ID, + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + }) + + views = [ + # Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc. + View(instrument_name="http.server.request.duration", + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS), + attribute_keys={"http.request.method", "http.route", "http.response.status_code"}), + View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + # No catch-all View("*"): an instrument matched by two views would be exported twice. + # Instruments without a matching view get the SDK default aggregation. + ] + meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())], + views=views) + metrics.set_meter_provider(meter_provider) + + tracer_provider = TracerProvider(resource=res, sampler=_build_sampler()) + tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter()))) + trace.set_tracer_provider(tracer_provider) + + logger_provider = LoggerProvider(resource=res) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + _setup_logging(logger_provider) + _state["providers"] = [tracer_provider, meter_provider, logger_provider] + + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + + # PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s + # placeholders), so values, message bodies and hashes never reach db.query.text. + BotocoreInstrumentor().instrument() + if fastapi_app is not None: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + # Health probes (LB every second) are excluded from spans and from the HTTP SLI. + FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready", + exclude_spans=["receive", "send"]) + _process_metrics() + logging.getLogger("messenger.telemetry").info( + "telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name, + os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0")) + + +def shutdown() -> None: + for provider in _state["providers"]: + try: + provider.shutdown() + except Exception: + pass + + +# ---------------------------------------------------------------------------------------------- context carrier + +def inject() -> dict: + """W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}.""" + carrier: dict = {} + if trace.get_current_span().get_span_context().is_valid: + propagate.inject(carrier) + return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")} + + +def extract(carrier: Optional[dict]): + """Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context.""" + if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))): + return otel_context.Context() + clean = {"traceparent": str(carrier["traceparent"])} + if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512: + clean["tracestate"] = carrier["tracestate"] + return propagate.extract(clean, context=otel_context.Context()) + + +def set_attributes(**attrs) -> None: + span = trace.get_current_span() + for key, value in attrs.items(): + if value is not None: + span.set_attribute(key.replace("__", "."), value) + + +@contextmanager +def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None): + with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {}, + record_exception=True, set_status_on_exception=True) as current: + yield current + + +def record_exception(exc: BaseException) -> None: + """Exception event on the active span (type + stack, never request data) for errors handled by our code.""" + current = trace.get_current_span() + current.record_exception(exc) + current.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + + +def mark_error(current, description: str) -> None: + current.set_status(Status(StatusCode.ERROR, description)) + + +# ---------------------------------------------------------------------------------------------- instruments + +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 + + +def _rss_bytes() -> int: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE + except OSError: + return 0 + + +def _process_metrics() -> None: + from opentelemetry.metrics import Observation + + def cpu(_options): + own = resource.getrusage(resource.RUSAGE_SELF) + kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit) + yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"}) + yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"}) + + meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)") + meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By", + description="Resident set size of the process") + + +def register_pool(db) -> None: + """DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats).""" + if not ENABLED: + return + from opentelemetry.metrics import Observation + + def count(_options): + stats = db.pool.get_stats() + size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0) + yield Observation(max(0, size - idle), {"db.client.connection.state": "used"}) + yield Observation(idle, {"db.client.connection.state": "idle"}) + + meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}") + meter.create_observable_up_down_counter( + "db.client.connection.pending_requests", + [lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}") + meter.create_observable_up_down_counter( + "db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}") + + +def register_queue(state: dict) -> None: + """Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second. + + Reported by every API replica (identical values): dashboards and alerts use max() across replicas. + """ + if not ENABLED: + return + from opentelemetry.metrics import Observation + + def depth(_options): + queue = state.get("queue") + if queue is None: + return [] + return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}), + Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}), + Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})] + + def oldest(_options): + queue = state.get("queue") + return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})] + + meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}", + description="Jobs accepted but not finished, by state") + meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s", + description="Age of the oldest queued/processing job since durable acceptance") + + +def register_worker(worker) -> None: + if not ENABLED: + return + from opentelemetry.metrics import Observation + + meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]], + unit="{job}", description="Jobs being processed right now") + meter.create_observable_gauge("messenger.worker.capacity", + [lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}", + description="Concurrent job slots of this worker") + + +jobs_processed = meter.create_counter( + "messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome") +job_duration = meter.create_histogram( + "messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome") +time_to_ready = meter.create_histogram( + "messenger.attachment.time_to_ready", unit="s", + description="From durable acceptance (job COMMIT, 202) to the ready COMMIT") diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..e65f800 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,373 @@ +"""Attachment worker: a separate process consuming attachments.process (python -m app.worker). + +Per delivery: + 1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ + 2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op + 3. download staging object, verify size+sha256 against DB metadata + 4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants + 5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes) + 6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner + 7. delete staging (best effort, reconciler finishes it), then ack +Ack happens only after the durable effect (6) or after the failure/retry decision is committed. +A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases. +""" +import asyncio +import hashlib +import json +import logging +import os +import resource +import signal +import sys +import tempfile +import time +import uuid +from datetime import datetime, timezone + +from opentelemetry.trace import SpanKind + +from app import config, faults, jobs, openbao, telemetry +from app.broker import connect, get_queue +from app.db import Database +from app.s3 import S3Storage + +log = logging.getLogger("messenger.worker") + +PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"} + + +class Permanent(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail or code + + +class Transient(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail or code + + +class LeaseLost(Exception): + pass + + +def child_limits() -> None: + memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + cpu = config.JOB_TIMEOUT_SECONDS + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + os.nice(5) + + +def sha256_file(path: str): + digest, size = hashlib.sha256(), 0 + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +class Worker: + def __init__(self) -> None: + self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8]) + self.db = Database() + self.s3 = S3Storage() + self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY) + self.stopping = asyncio.Event() + self.inflight = 0 + self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0} + + # ------------------------------------------------------------------ lifecycle + + async def run(self) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self.stopping.set) + await self.db.open() + telemetry.register_pool(self.db) + telemetry.register_worker(self) + await asyncio.to_thread(self.s3.open) + bao = openbao.client() + if bao is not None: + # Rotation of the worker's DB password / S3 key in OpenBao is picked up without a restart. + bao.on_change(lambda data: self.db.rotate(data.get("database_url", ""))) + bao.on_change(self.s3.rotate) + bao.start_refresher(config.BAO_REFRESH_SECONDS) + connection = await connect() + channel = await connection.channel() + await channel.set_qos(prefetch_count=config.WORKER_PREFETCH) + queue = await get_queue(channel) + tag = await queue.consume(self.on_message, no_ack=False) + background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())] + log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB", + self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS, + config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB) + await self.stopping.wait() + log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight) + try: + await queue.cancel(tag) + except Exception: + pass + deadline = time.monotonic() + config.WORKER_GRACE_SECONDS + while self.inflight and time.monotonic() < deadline: + await asyncio.sleep(0.2) + for task in background: + task.cancel() + # Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed. + await connection.close() + await self.db.close() + log.info("worker_stopped stats=%s", self.stats) + + async def alive(self) -> None: + while True: + try: + with open(config.WORKER_HEARTBEAT_FILE, "w") as handle: + handle.write(str(time.time())) + except OSError: + pass + await asyncio.sleep(2) + + async def sweeper(self) -> None: + while True: + await asyncio.sleep(config.JOB_SWEEP_SECONDS) + try: + swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner)) + if swept: + log.warning("lease_expired recovered=%s", swept) + except Exception: + log.warning("lease_sweep_skipped: database unavailable") + + # ------------------------------------------------------------------ delivery + + async def on_message(self, message) -> None: + async with self.slots: + self.inflight += 1 + started = time.monotonic() + # CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier). + try: + carrier = json.loads(message.body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + carrier = None + if not isinstance(carrier, dict) or "traceparent" not in carrier: + carrier = dict(message.headers or {}) + attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process", + "messaging.destination.name": config.BROKER_QUEUE, + "messaging.message.id": str(message.message_id or ""), + "messaging.rabbitmq.message.redelivered": bool(message.redelivered)} + outcome = "requeued" + with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER, + context=telemetry.extract(carrier), attributes=attrs) as current: + try: + outcome = await self.handle(message) + except Exception: + log.exception("unexpected_error, message returned to queue") + await asyncio.sleep(1) + await self.settle(message.nack(requeue=True)) + finally: + self.inflight -= 1 + current.set_attribute("messenger.job.outcome", outcome) + if outcome in ("retry", "failed", "dlq", "requeued", "lost"): + telemetry.mark_error(current, outcome) + labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome} + telemetry.jobs_processed.add(1, labels) + telemetry.job_duration.record(time.monotonic() - started, labels) + + @staticmethod + async def settle(awaitable) -> None: + try: + await awaitable + except Exception: + log.warning("could not settle delivery (connection lost); broker will redeliver") + + async def handle(self, message) -> str: + """Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued.""" + msg = jobs.parse_message(message.body) + if msg is None: + self.stats["dlq"] += 1 + log.error("job_invalid_message -> DLQ message_id=%s", message.message_id) + await self.settle(message.reject(requeue=False)) + return "dlq" + telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"]) + try: + decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner)) + except Exception: + log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"]) + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return "requeued" + if decision == "done" and job["state"] == "succeeded": + await self.cleanup_staging(job["attachment_id"]) + if decision != "claimed": + self.stats["noop"] += 1 + telemetry.set_attributes(messenger__job__decision=decision) + log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision, + message.redelivered) + await self.settle(message.ack()) + return "noop" + started = time.monotonic() + log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"], + job["max_attempts"]) + lease_lost = asyncio.Event() + keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost)) + outcome = None + work = asyncio.create_task(self.process(job, message)) + lost = asyncio.create_task(lease_lost.wait()) + try: + done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2, + return_when=asyncio.FIRST_COMPLETED) + if work not in done: + work.cancel() + raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout") + outcome = "succeeded" if work.result() else "lost" + except LeaseLost: + outcome = "lost" + except Permanent as exc: + outcome = await self.record_failure(job, exc.code, exc.detail, transient=False) + except Transient as exc: + outcome = await self.record_failure(job, exc.code, exc.detail, transient=True) + except Exception as exc: # S3/DB/network errors + outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True) + finally: + keeper.cancel() + lost.cancel() + if outcome is None: + # Could not record the decision: keep the message; the lease expires and the sweeper takes over. + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return "requeued" + self.stats[outcome] = self.stats.get(outcome, 0) + 1 + log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome, + time.monotonic() - started) + await self.settle(message.ack()) + return outcome + + async def cleanup_staging(self, attachment_id) -> None: + """Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup.""" + try: + async with self.db.connection() as conn: + src = await jobs.attachment_source(conn, attachment_id) + if src and src["status"] == "ready" and src["staging_key"]: + await asyncio.to_thread(self.s3.delete, src["staging_key"]) + except Exception: + log.warning("staging_cleanup_deferred attachment=%s", attachment_id) + + async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None: + interval = max(1.0, config.JOB_LEASE_SECONDS / 3) + while True: + await asyncio.sleep(interval) + try: + alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner)) + except Exception: + continue # DB blip: keep trying until the lease really expires + if not alive: + log.warning("lease_lost job=%s: another worker owns it now", job_id) + lease_lost.set() + return + + async def record_failure(self, job, code: str, detail: str, transient: bool): + log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code, + transient) + try: + return await self.db.in_transaction( + lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient)) + except Exception: + log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"]) + return None + + # ------------------------------------------------------------------ the actual work + + async def process(self, job, message) -> bool: + async with self.db.connection() as conn: + src = await jobs.attachment_source(conn, job["attachment_id"]) + if src is None or not src["staging_key"]: + raise Permanent("staging_missing", "attachment has no staging object") + filename = src["filename"] + first_delivery = job["attempt"] == 1 and not message.redelivered + if faults.active(filename, "worker_hang") and job["attempt"] == 1: + log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5)) + time.sleep(config.JOB_LEASE_SECONDS * 2.5) + if faults.active(filename, "permanent"): + raise Permanent("invalid_image", "injected permanent failure") + if job["attempt"] <= faults.transient_until(filename): + raise Transient("dependency_unavailable", "injected transient failure") + + with tempfile.TemporaryDirectory(prefix="job-") as tmp: + source = os.path.join(tmp, "source") + if not await asyncio.to_thread(self.s3.download, src["staging_key"], source): + raise Permanent("staging_missing", "staging object not found") + digest, size = await asyncio.to_thread(sha256_file, source) + if digest != src["sha256"] or size != src["size_bytes"]: + raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload") + out_dir = os.path.join(tmp, "out") + os.mkdir(out_dir) + with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}): + produced = await self.run_child(source, out_dir) + + variants = [] + prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"]) + await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"]) + for item in produced: + digest, size = await asyncio.to_thread(sha256_file, item["file"]) + key = prefix + os.path.basename(item["file"]) + await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"]) + variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"], + "size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest}) + if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS): + raise Permanent("processing_failed", "not all variants were produced") + + if first_delivery and faults.active(filename, "worker_crash_after_put"): + faults.crash("after S3 put, before result COMMIT") + committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants)) + if not committed: + log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"]) + return False + telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()), + {"job.type": telemetry.JOB_TYPE}) + if first_delivery and faults.active(filename, "worker_crash_after_commit"): + faults.crash("after result COMMIT, before ack") + try: + await asyncio.to_thread(self.s3.delete, src["staging_key"]) + except Exception: + log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"]) + return True + + async def run_child(self, source: str, out_dir: str) -> list: + env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app", + "JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)} + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS), + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits, + ) + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None + except asyncio.CancelledError: + proc.kill() + raise + code = proc.returncode + if code == 0: + return json.loads(stdout) + if code in PERMANENT_EXIT: + raise Permanent(PERMANENT_EXIT[code]) + if code == -signal.SIGXCPU: + raise Permanent("image_too_complex", "CPU limit exceeded") + if code == -signal.SIGKILL: + raise Transient("worker_resource_exhausted", "child killed (OOM)") + raise Permanent("processing_failed", "child exit %s" % code) + + +def main() -> None: + telemetry.setup("messenger-worker") + try: + asyncio.run(Worker().run()) + finally: + telemetry.shutdown() + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3088dee --- /dev/null +++ b/compose.yaml @@ -0,0 +1,604 @@ +name: mtusi-lab7 +# Trust layout: contracts/trust-matrix.md. Every TLS key reaches only its own container (Docker secrets from +# secrets/pki, created by scripts/pki.sh). API and worker receive NO passwords: they authenticate to OpenBao with +# their client certificate and read their scoped KV secret (scripts/bao.sh bootstrap). + +x-hardened: &hardened + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + read_only: true + restart: unless-stopped + +x-otel-env: &otel-env + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca.crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel.crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel.key + OTEL_EXPORTER_OTLP_TIMEOUT: "5" + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_SEMCONV_STABILITY_OPT_IN: http + OTEL_METRIC_EXPORT_INTERVAL: "10000" + OTEL_BSP_MAX_QUEUE_SIZE: "2048" + OTEL_BSP_SCHEDULE_DELAY: "2000" + OTEL_BLRP_MAX_QUEUE_SIZE: "2048" + SERVICE_VERSION: "1.7.0" + +x-app-env: &app-env + TLS_CA_FILE: /run/secrets/ca.crt + TLS_CLIENT_CERT_FILE: /run/secrets/client.crt + TLS_CLIENT_KEY_FILE: /run/secrets/client.key + BAO_ADDR: https://openbao:8200 + S3_ENDPOINT: https://minio:9000 + BROKER_URL: amqps://rabbitmq:5671/messenger?auth=external&heartbeat=30 + DATABASE_SSLMODE: verify-full + +x-api: &api + <<: *hardened + build: + context: . + image: mtusi-messenger:lab7 + env_file: [.env] + tmpfs: ["/tmp:size=128m,mode=1777"] + depends_on: + postgres: {condition: service_healthy} + rabbitmq: {condition: service_healthy} + valkey: {condition: service_healthy} + openbao: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + migrate: {condition: service_completed_successfully} + deploy: + resources: + limits: {cpus: "0.50", memory: 512M, pids: 256} + # Business port 8080 is mTLS-only; the healthcheck uses the loopback-only health listener 127.0.0.1:8081. + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + +services: + nginx: + <<: *hardened + image: nginx:1.30.6-alpine@sha256:583b7f2fce1cdb8da5ad5c39ebd3c33b98b87d493190e548bdee1fa8c619c311 + user: "101:101" + command: ["/bin/sh", "/opt/nginx-health.sh"] + environment: + API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2} + HEALTH_INTERVAL: "1" + HEALTH_FALL: "2" + HEALTH_RISE: "2" + ports: + - "127.0.0.1:8443:8443" # the only business ingress: TLS + tmpfs: ["/tmp:uid=101,gid=101,mode=0700", "/var/cache/nginx:uid=101,gid=101,mode=0700", + "/var/run:uid=101,gid=101,mode=0755", "/etc/nginx/upstream:uid=101,gid=101,mode=0755"] + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx-health.sh:/opt/nginx-health.sh:ro + secrets: + - {source: ca_crt, target: ca.crt} + - {source: lb_server_crt, target: server.crt} + - {source: lb_server_key, target: server.key} + - {source: lb_client_crt, target: client.crt} + - {source: lb_client_key, target: client.key} + depends_on: + api-1: {condition: service_healthy} + api-2: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.25", memory: 64M, pids: 256} + healthcheck: + test: ["CMD-SHELL", "curl -sf --cacert /run/secrets/ca.crt https://localhost:8443/health/live >/dev/null"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + + api-1: + <<: *api + environment: + <<: [*otel-env, *app-env] + INSTANCE_ID: api-1 + LOGIN_RATE_LIMIT_PER_IP: ${LOGIN_RATE_LIMIT_PER_IP:-120} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4} + BAO_AUTH_ROLE: api + BAO_KV_PATH: api + TLS_SERVER_CERT_FILE: /run/secrets/server.crt + TLS_SERVER_KEY_FILE: /run/secrets/server.key + ALLOWED_CLIENT_IDENTITIES: lb.client.messenger + REVOKED_SERIALS_FILE: /run/secrets/revoked-serials.txt + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + secrets: &api-secrets + - {source: ca_crt, target: ca.crt} + - {source: api_1_crt, target: server.crt} + - {source: api_1_key, target: server.key} + - {source: api_client_crt, target: client.crt} + - {source: api_client_key, target: client.key} + - {source: api_otel_crt, target: otel.crt} + - {source: api_otel_key, target: otel.key} + - {source: revoked_serials, target: revoked-serials.txt} + + api-2: + <<: *api + environment: + <<: [*otel-env, *app-env] + INSTANCE_ID: api-2 + LOGIN_RATE_LIMIT_PER_IP: ${LOGIN_RATE_LIMIT_PER_IP:-120} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4} + BAO_AUTH_ROLE: api + BAO_KV_PATH: api + TLS_SERVER_CERT_FILE: /run/secrets/server.crt + TLS_SERVER_KEY_FILE: /run/secrets/server.key + ALLOWED_CLIENT_IDENTITIES: lb.client.messenger + REVOKED_SERIALS_FILE: /run/secrets/revoked-serials.txt + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + secrets: + - {source: ca_crt, target: ca.crt} + - {source: api_2_crt, target: server.crt} + - {source: api_2_key, target: server.key} + - {source: api_client_crt, target: client.crt} + - {source: api_client_key, target: client.key} + - {source: api_otel_crt, target: otel.crt} + - {source: api_otel_key, target: otel.key} + - {source: revoked_serials, target: revoked-serials.txt} + + worker: + <<: *hardened + build: + context: . + image: mtusi-messenger:lab7 + command: ["python", "-m", "app.worker"] + env_file: [.env] + environment: + <<: [*otel-env, *app-env] + INSTANCE_ID: "" + BAO_AUTH_ROLE: worker + BAO_KV_PATH: worker + DB_POOL_MAX: "6" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + tmpfs: ["/tmp:size=512m,mode=1777"] + secrets: + - {source: ca_crt, target: ca.crt} + - {source: worker_client_crt, target: client.crt} + - {source: worker_client_key, target: client.key} + - {source: worker_otel_crt, target: otel.crt} + - {source: worker_otel_key, target: otel.key} + depends_on: + postgres: {condition: service_healthy} + rabbitmq: {condition: service_healthy} + openbao: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + migrate: {condition: service_completed_successfully} + stop_grace_period: 30s + deploy: + replicas: ${WORKER_REPLICAS:-2} + resources: + limits: {cpus: "${WORKER_CPUS:-2.00}", memory: 1536M, pids: 256} + healthcheck: + test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + + migrate: + <<: *hardened + build: + context: . + image: mtusi-messenger:lab7 + command: ["python", "scripts/migrate.py"] + restart: "no" + environment: + DATABASE_URL_FILE: /run/secrets/migrate_database_url + DATABASE_SSLMODE: verify-full + TLS_CA_FILE: /run/secrets/ca.crt + DB_API_PASSWORD_FILE: /run/secrets/db_api_password + DB_WORKER_PASSWORD_FILE: /run/secrets/db_worker_password + secrets: + - {source: ca_crt, target: ca.crt} + - migrate_database_url + - db_api_password + - db_worker_password + depends_on: + postgres: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.25", memory: 128M, pids: 256} + + openbao: + <<: *hardened + image: openbao/openbao:2.6.5@sha256:09f1eaaee351c329d75a5a89654d6e1ad4792998650c3cbf0c3513a76d9d9b6e + user: "100:1000" + entrypoint: ["bao", "server", "-config=/openbao/config/config.hcl"] + tmpfs: ["/tmp"] + volumes: + - ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro + - baodata:/openbao/file + secrets: + - {source: ca_crt, target: ca.crt} + - {source: openbao_crt, target: openbao.crt} + - {source: openbao_key, target: openbao.key} + deploy: + resources: + limits: {cpus: "0.50", memory: 256M, pids: 256} + # Healthy = server answers (sealed or not). Services retry their login while it is sealed. + healthcheck: + test: ["CMD-SHELL", "BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/run/secrets/ca.crt bao status >/dev/null 2>&1; [ $$? -ne 1 ]"] + interval: 5s + timeout: 3s + retries: 20 + + postgres: + <<: *hardened + image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2 + user: "70:70" + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: --data-checksums --auth-host=scram-sha-256 + # PostgreSQL refuses a key file readable by others: copy it to the tmpfs with mode 0600 first. + entrypoint: ["sh", "-c", "install -m 600 /run/secrets/server.key /tmp/server.key && exec docker-entrypoint.sh \"$$@\"", "--"] + command: + - postgres + - -c + - ssl=on + - -c + - ssl_cert_file=/run/secrets/server.crt + - -c + - ssl_key_file=/tmp/server.key + - -c + - ssl_min_protocol_version=TLSv1.2 + - -c + - hba_file=/etc/postgresql/pg_hba.conf + - -c + - password_encryption=scram-sha-256 + - -c + - synchronous_commit=on + - -c + - shared_buffers=128MB + - -c + - max_connections=60 + tmpfs: ["/tmp", "/var/run/postgresql"] + secrets: + - postgres_password + - {source: postgres_crt, target: server.crt} + - {source: postgres_key, target: server.key} + volumes: + - pgdata:/var/lib/postgresql/data + - ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + deploy: + resources: + limits: {cpus: "1.00", memory: 512M, pids: 256} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h /var/run/postgresql"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + + valkey: + <<: *hardened + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + user: "999:1000" + entrypoint: ["sh", "/opt/valkey-start.sh"] + tmpfs: ["/tmp", "/data"] + volumes: + - ./infra/valkey/valkey-start.sh:/opt/valkey-start.sh:ro + secrets: + - {source: ca_crt, target: ca.crt} + - {source: valkey_crt, target: server.crt} + - {source: valkey_key, target: server.key} + - valkey_api_password + deploy: + resources: + limits: {cpus: "0.25", memory: 128M, pids: 256} + healthcheck: + test: ["CMD", "valkey-cli", "--tls", "--cacert", "/run/secrets/ca.crt", "--user", "health", "--pass", "health", "-h", "valkey", "ping"] + interval: 5s + timeout: 3s + retries: 10 + + rabbitmq: + <<: *hardened + image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32 + user: "100:101" + hostname: rabbitmq + environment: + RABBITMQ_ENABLED_PLUGINS_FILE: /etc/rabbitmq/enabled_plugins + # Erlang distribution + epmd loopback-only (no cluster; CLI runs inside the container). The node name must + # then resolve to loopback, hence rabbit@localhost (its data dir is /var/lib/rabbitmq/mnesia/rabbit@localhost). + RABBITMQ_NODENAME: rabbit@localhost + ERL_EPMD_ADDRESS: 127.0.0.1 + tmpfs: ["/tmp", "/etc/rabbitmq/conf.d"] + volumes: + - rabbitmqdata:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + - ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro + - ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro + secrets: + - {source: ca_crt, target: ca.crt} + - {source: rabbitmq_crt, target: server.crt} + - {source: rabbitmq_key, target: server.key} + deploy: + resources: + limits: {cpus: "1.00", memory: 512M, pids: 256} + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 5s + timeout: 10s + retries: 30 + start_period: 20s + + minio: + <<: *hardened + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + command: ["server", "/data", "--certs-dir", "/tmp/certs"] + entrypoint: ["sh", "-c", "mkdir -p /tmp/certs/CAs && cp /run/secrets/server.crt /tmp/certs/public.crt && cp /run/secrets/server.key /tmp/certs/private.key && cp /run/secrets/ca.crt /tmp/certs/CAs/ca.crt && exec minio \"$$@\"", "--"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + tmpfs: ["/tmp"] + secrets: + - minio_root_user + - minio_root_password + - {source: ca_crt, target: ca.crt} + - {source: minio_crt, target: server.crt} + - {source: minio_key, target: server.key} + volumes: + - miniodata:/data + deploy: + resources: + limits: {cpus: "0.50", memory: 512M, pids: 256} + healthcheck: + test: ["CMD-SHELL", "mkdir -p /tmp/hc/certs/CAs && cp /run/secrets/ca.crt /tmp/hc/certs/CAs/ca.crt && MC_HOST_hc=https://minio:9000 mc --config-dir /tmp/hc ready hc >/dev/null"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + + minio-init: + <<: *hardened + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + entrypoint: ["/bin/sh", "/opt/init.sh"] + restart: "no" + environment: + S3_BUCKET: messenger-attachments + MC_CONFIG_DIR: /tmp/mc + tmpfs: ["/tmp"] + secrets: + - minio_root_user + - minio_root_password + - {source: ca_crt, target: ca.crt} + - s3_api_access_key + - s3_api_secret_key + - s3_worker_access_key + - s3_worker_secret_key + volumes: + - ./infra/minio/init.sh:/opt/init.sh:ro + - ./infra/minio/policy-api.json:/etc/minio-policy/policy-api.json:ro + - ./infra/minio/policy-worker.json:/etc/minio-policy/policy-worker.json:ro + depends_on: + minio: {condition: service_healthy} + + # ------------------------------------------------------------------ observability (lab6) behind mTLS + otel-collector: + <<: *hardened + image: otel/opentelemetry-collector-contrib:0.120.0@sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917 + command: + - --config=/etc/otelcol/collector.yaml + - --config=/etc/otelcol/profile-${COLLECTOR_PROFILE:-default}.yaml + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + - ./infra/otel/profile-default.yaml:/etc/otelcol/profile-default.yaml:ro + - ./infra/otel/profile-tail.yaml:/etc/otelcol/profile-tail.yaml:ro + secrets: + - {source: ca_crt, target: ca.crt} + - {source: ingest_ca_crt, target: ingest-ca.crt} + - {source: otel_collector_crt, target: server.crt} + - {source: otel_collector_key, target: server.key} + - {source: collector_out_crt, target: client.crt} + - {source: collector_out_key, target: client.key} + deploy: + resources: + limits: {cpus: "1.00", memory: 256M, pids: 256} + + prometheus: + <<: *hardened + image: prom/prometheus:v3.2.1@sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.config.file=/etc/prometheus/web.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --web.enable-otlp-receiver + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro + - ./infra/prometheus/health-client.yml:/etc/prometheus/health-client.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - promdata:/prometheus + secrets: + - {source: ca_crt, target: ca.crt} + - {source: telemetry_ca_crt, target: telemetry-ca.crt} + - {source: prometheus_crt, target: server.crt} + - {source: prometheus_key, target: server.key} + - {source: operator_crt, target: health.crt} + - {source: operator_key, target: health.key} + ports: + - "127.0.0.1:9090:9090" # mTLS: operator certificate only + deploy: + resources: + limits: {cpus: "1.00", memory: 512M, pids: 256} + healthcheck: + test: ["CMD", "promtool", "check", "healthy", "--url=https://prometheus:9090", "--http.config.file=/etc/prometheus/health-client.yml"] + interval: 5s + timeout: 3s + retries: 20 + + loki: + <<: *hardened + image: grafana/loki:3.4.2@sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + secrets: + - {source: telemetry_ca_crt, target: telemetry-ca.crt} + - {source: loki_crt, target: server.crt} + - {source: loki_key, target: server.key} + ports: + - "127.0.0.1:3100:3100" # mTLS: operator certificate only + deploy: + resources: + limits: {cpus: "1.00", memory: 512M, pids: 256} + healthcheck: # TCP only: every HTTP request needs a telemetry client certificate + test: ["CMD", "/busybox/nc", "-z", "127.0.0.1", "3100"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + + tempo: + <<: *hardened + image: grafana/tempo:2.7.1@sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69 + command: ["-config.file=/etc/tempo/tempo.yaml"] + tmpfs: ["/tmp"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + secrets: + - {source: telemetry_ca_crt, target: telemetry-ca.crt} + - {source: tempo_crt, target: server.crt} + - {source: tempo_key, target: server.key} + ports: + - "127.0.0.1:3200:3200" # mTLS: operator certificate only + deploy: + resources: + limits: {cpus: "1.00", memory: 1024M, pids: 256} + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "3200"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + + grafana: + <<: *hardened + image: grafana/grafana:11.5.2@sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7 + environment: + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /run/secrets/server.crt + GF_SERVER_CERT_KEY: /run/secrets/server.key + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_SECURITY_COOKIE_SECURE: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + tmpfs: ["/tmp"] + secrets: + - grafana_admin_password + - {source: ca_crt, target: ca.crt} + - {source: grafana_crt, target: server.crt} + - {source: grafana_key, target: server.key} + - {source: grafana_out_crt, target: client.crt} + - {source: grafana_out_key, target: client.key} + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" # HTTPS + admin login + depends_on: + prometheus: {condition: service_healthy} + loki: {condition: service_started} + tempo: {condition: service_started} + deploy: + resources: + limits: {cpus: "0.50", memory: 256M, pids: 256} + healthcheck: + test: ["CMD-SHELL", "curl -sf --cacert /run/secrets/ca.crt https://grafana:3000/api/health >/dev/null"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + +volumes: + pgdata: + miniodata: + rabbitmqdata: + baodata: + promdata: + lokidata: + tempodata: + grafanadata: + +secrets: + ca_crt: {file: ./secrets/pki/ca.crt} + ingest_ca_crt: {file: ./secrets/pki/ingest.crt} + telemetry_ca_crt: {file: ./secrets/pki/telemetry.crt} + revoked_serials: {file: ./secrets/pki/revoked-serials.txt} + lb_server_crt: {file: ./secrets/pki/lb-server.crt} + lb_server_key: {file: ./secrets/pki/lb-server.key} + lb_client_crt: {file: ./secrets/pki/lb-client.crt} + lb_client_key: {file: ./secrets/pki/lb-client.key} + api_1_crt: {file: ./secrets/pki/api-1.crt} + api_1_key: {file: ./secrets/pki/api-1.key} + api_2_crt: {file: ./secrets/pki/api-2.crt} + api_2_key: {file: ./secrets/pki/api-2.key} + api_client_crt: {file: ./secrets/pki/api.crt} + api_client_key: {file: ./secrets/pki/api.key} + worker_client_crt: {file: ./secrets/pki/worker.crt} + worker_client_key: {file: ./secrets/pki/worker.key} + api_otel_crt: {file: ./secrets/pki/api-otel.crt} + api_otel_key: {file: ./secrets/pki/api-otel.key} + worker_otel_crt: {file: ./secrets/pki/worker-otel.crt} + worker_otel_key: {file: ./secrets/pki/worker-otel.key} + postgres_crt: {file: ./secrets/pki/postgres.crt} + postgres_key: {file: ./secrets/pki/postgres.key} + minio_crt: {file: ./secrets/pki/minio.crt} + minio_key: {file: ./secrets/pki/minio.key} + rabbitmq_crt: {file: ./secrets/pki/rabbitmq.crt} + rabbitmq_key: {file: ./secrets/pki/rabbitmq.key} + valkey_crt: {file: ./secrets/pki/valkey.crt} + valkey_key: {file: ./secrets/pki/valkey.key} + openbao_crt: {file: ./secrets/pki/openbao.crt} + openbao_key: {file: ./secrets/pki/openbao.key} + otel_collector_crt: {file: ./secrets/pki/otel-collector.crt} + otel_collector_key: {file: ./secrets/pki/otel-collector.key} + collector_out_crt: {file: ./secrets/pki/collector-out.crt} + collector_out_key: {file: ./secrets/pki/collector-out.key} + prometheus_crt: {file: ./secrets/pki/prometheus.crt} + prometheus_key: {file: ./secrets/pki/prometheus.key} + loki_crt: {file: ./secrets/pki/loki.crt} + loki_key: {file: ./secrets/pki/loki.key} + tempo_crt: {file: ./secrets/pki/tempo.crt} + tempo_key: {file: ./secrets/pki/tempo.key} + grafana_crt: {file: ./secrets/pki/grafana.crt} + grafana_key: {file: ./secrets/pki/grafana.key} + grafana_out_crt: {file: ./secrets/pki/grafana-out.crt} + grafana_out_key: {file: ./secrets/pki/grafana-out.key} + operator_crt: {file: ./secrets/pki/operator.crt} + operator_key: {file: ./secrets/pki/operator.key} + postgres_password: {file: ./secrets/bootstrap/postgres_password} + migrate_database_url: {file: ./secrets/bootstrap/migrate_database_url} + db_api_password: {file: ./secrets/bootstrap/db_api_password} + db_worker_password: {file: ./secrets/bootstrap/db_worker_password} + minio_root_user: {file: ./secrets/bootstrap/minio_root_user} + minio_root_password: {file: ./secrets/bootstrap/minio_root_password} + s3_api_access_key: {file: ./secrets/bootstrap/s3_api_access_key} + s3_api_secret_key: {file: ./secrets/bootstrap/s3_api_secret_key} + s3_worker_access_key: {file: ./secrets/bootstrap/s3_worker_access_key} + s3_worker_secret_key: {file: ./secrets/bootstrap/s3_worker_secret_key} + valkey_api_password: {file: ./secrets/bootstrap/valkey_api_password} + grafana_admin_password: {file: ./secrets/bootstrap/grafana_admin_password} diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..0a943c4 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -4,18 +4,18 @@ | Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | `infra/nginx.conf` (listen 8443 ssl), `app/main.py` login; аудит I1–I4, `make test-security`, `test_auth_acl.py` | +| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | `app/serve.py` (CERT_REQUIRED + SAN allowlist + отозванные serial), `infra/nginx.conf` proxy_ssl_*; аудит A1–A14, `test_cert_rotation.py` | +| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | `infra/postgres/pg_hba.conf`, `migrations/0007_service_roles.sql`, `app/db.py` (verify-full); аудит B1–B14, `test_secret_rotation.py` | +| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | `infra/minio/policy-*.json`, `app/s3.py` (verify=CA); аудит C1–C14 | +| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | `infra/rabbitmq/rabbitmq.conf`, `definitions.json`, `app/broker.py`; аудит D1–D14 | +| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | `infra/otel/collector.yaml` (client_ca_file = ingest CA); аудит E1–E7 | +| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | `infra/prometheus/web.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, exporters в `collector.yaml`; аудит F1–F24 | +| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | `infra/grafana/provisioning/datasources/datasources.yaml` (tlsAuth + CA); аудит F (grafana-out), E (grafana-out отклонён), J1–J2 | +| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | `infra/openbao/config.hcl`, `policies/*.hcl`, `scripts/bao.sh`, `app/openbao.py`; аудит H1–H20, `test_bao_sealed.py` | +| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | `scripts/bao.sh`, `compose.yaml` (порты только 127.0.0.1); аудит J, H (admin-пути запрещены приложениям), `check_hardening.py` | +| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | `infra/valkey/valkey-start.sh` (TLS, ACL), `app/sessions.py`; аудит G1–G11 | +| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | N/A: репликации и кластеров нет (по одному узлу каждого хранилища). Erlang distribution RabbitMQ, внутренний gRPC Loki/Tempo — только loopback; аудит K1–K13 | Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. @@ -35,3 +35,64 @@ OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. + + +--- + +# Заполнение (решение lab7) + +Все отрицательные и положительные проверки ниже выполняет `student-tests/tls_audit.py` изнутри сети стенда +(разделы A–K, **135 проверок**). Сырой вывод: `evidence/tls_audit.txt`, `evidence/tls_audit.json`. +Запуск: `docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro -v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py`. + +## PKI и identities + +| CA | Кем подписан | Что подписывает | Кто доверяет | +| --- | --- | --- | --- | +| Messenger Lab Root CA (`ca.crt`, EC P-256, 825 дн.) | self-signed | все серверные сертификаты; клиентские identities `lb`, `api`, `worker`, `intruder` (тестовый) | все клиенты для проверки сервера; API (LB), RabbitMQ, OpenBao, PostgreSQL | +| Telemetry Ingest CA | Root | только `api.otel.messenger`, `worker.otel.messenger` | **только** Collector (client_ca_file) | +| Telemetry Clients CA | Root | только `collector.out`, `grafana.out`, `operator` | **только** Prometheus, Loki, Tempo | +| Foreign CA (тестовый) | self-signed | `foreign-client`, заявляет имена lb/api/worker | никто | + +Leaf-сертификаты живут 90 дней (`scripts/pki.sh`), ключи EC P-256 лежат только в `secrets/pki/` (0700) и попадают в контейнер как Docker secret именно того сервиса. Отзыв: `revoked-serials.txt`, API перечитывает его на лету. + +## Детальная матрица + +| Связь | Порт / endpoint | Сервер (CA, SAN) | Клиентская identity | Проверка SAN / ACL | Срок жизни credentials | Положительный / отрицательный тест | +| --- | --- | --- | --- | --- | --- | --- | +| Клиент → nginx | `127.0.0.1:8443` HTTPS | lb-server: Root, SAN `localhost`, `nginx`, `127.0.0.1` | пользователь: Bearer (256 бит, в БД SHA-256), вход Basic | сессия проверяется на каждом запросе (PostgreSQL, кэш Valkey ≤ 30 с и не дольше expires_at); logout = отзыв в кэше и БД; лимит входа 5 неудач на username и 120 попыток на IP за 60 с, общий для реплик (Valkey) | сессия 3600 с; сертификат 90 дн. | `make test`, `make test-security`; аудит I: чужой CA и неверный hostname → ошибка проверки сертификата; `test_auth_acl.py`: logout, expiry, 429 + Retry-After | +| nginx → API-1/2 | `api-N:8080` HTTPS, mTLS; plaintext-listener нет | api-N: Root, SAN `api-N`, `api.internal` (nginx проверяет `api.internal`) | `lb.client.messenger` (Root, EKU clientAuth) | `app/serve.py`: CERT_REQUIRED (цепочка до Root) **и** DNS SAN ∈ `ALLOWED_CLIENT_IDENTITIES` **и** serial ∉ revoked; проверяется на каждом соединении, включая возобновлённые TLS-сессии | 90 дн.; отзыв ≤ 1 с для новых соединений | аудит A: без сертификата, чужой CA, `intruder`/`worker`/`api` от Root → соединение закрыто; `lb` → 200. `test_cert_rotation.py`: ротация, отзыв старого serial, просроченный | +| API/worker → PostgreSQL | `postgres:5432`, `sslmode=verify-full` | postgres: Root, SAN `postgres` | роли `app_api`, `app_worker` (SCRAM-SHA-256) | `pg_hba`: только `hostssl … scram-sha-256`, `host … reject`; права по таблицам: worker без users/messages/sessions и без DELETE, API без DDL и ALTER ROLE | пароль в OpenBao KV; ротация `bao.sh rotate-worker-db` | аудит B: plaintext, чужой CA, неверный hostname, неверный пароль → отказ; 9 проверок allow/deny прав | +| API/worker → MinIO | `minio:9000` HTTPS | minio: Root, SAN `minio` | `messenger-api`, `messenger-worker` | API: PUT/GET/DELETE `staging/*`, GET/DELETE `attachments/*`, LIST; worker: GET/DELETE `staging/*`, только PUT `attachments/*` | ключи в OpenBao KV (callback `s3.rotate`) | аудит C: чужой CA, plaintext → отказ; worker: GET/DELETE `attachments`, PUT `staging`, LIST, чужой префикс → AccessDenied; API: PUT `attachments`, создание бакета → AccessDenied | +| API → RabbitMQ (producer) | `rabbitmq:5671` AMQPS, mTLS; 5672 закрыт | rabbitmq: Root, SAN `rabbitmq` | `api.client.messenger` (EXTERNAL = CN) | permissions: configure `^$`, write `^attachments$`, read `^$` | 90 дн. | аудит D: без сертификата, чужой CA, `intruder`, `lb` → отказ; API: publish разрешён; consume, declare и default exchange → ACCESS_REFUSED | +| Worker → RabbitMQ (consumer) | то же | то же | `worker.client.messenger` | configure `^$`, write `^$`, read `^attachments\.process$` | 90 дн. | worker: consume разрешён; publish и declare → ACCESS_REFUSED | +| API/worker → Collector | `otel-collector:4318` OTLP/HTTPS, mTLS; gRPC 4317 выключен | otel-collector: Root | `api.otel.messenger` / `worker.otel.messenger` (Ingest CA) | client_ca_file = только Ingest CA: сертификаты Root-CA и Telemetry Clients CA отклоняются | 90 дн. | аудит E: без сертификата, чужой CA, `lb`/`api` (Root), `grafana-out` → отказ; api-otel и worker-otel → 200 | +| Collector → Prometheus / Loki / Tempo | `prometheus:9090` `/api/v1/otlp`, `loki:3100` `/otlp`, `tempo:4317` gRPC — все с TLS | Root, SAN по имени сервиса | `collector.out.messenger` (Telemetry Clients CA) | бэкенды доверяют только Telemetry Clients CA | 90 дн. | аудит F: без сертификата, чужой CA, `api` (Root), `api-otel` (Ingest) → отказ; collector-out, grafana-out, operator → 200 | +| Grafana → источники | те же HTTPS-порты | проверка сервера по Root | `grafana.out.messenger` | Telemetry Clients CA; к Collector и API не допускается | 90 дн. | аудит F (accept), E (grafana-out в Collector → отказ); запросы панелей через Grafana: `evidence/dashboard_check.txt` | +| API/worker → OpenBao | `openbao:8200` HTTPS | openbao: Root, SAN `openbao` | сертификат `api` / `worker` → cert auth (роль = ровно этот CN) | политика `messenger-`: только `read` своего `messenger/data/` и metadata | токен 15 мин, max 1 ч; renew/re-login каждые 15 с | аудит H: без сертификата, `intruder`, чужая роль, чужой CA → нет токена; чужой путь, запись, изменение политики, admin-пути → 403 | +| Оператор → OpenBao | не опубликован; `docker compose exec` + CLI по TLS на 127.0.0.1 | Root | root token и 3 доли unseal в `secrets/operator/` (0600) | у приложений нет admin-прав (аудит H: `sys/auth` → 403) | root token живёт у оператора; доли нужны при каждом рестарте OpenBao | `scripts/bao.sh status|unseal|seal` | +| Оператор → Grafana | `127.0.0.1:3000` HTTPS | grafana: Root, SAN `localhost` | admin-пароль (Docker secret) | anonymous выключен, регистрация выключена | — | аудит J: без логина → 401; plaintext → 400 | +| Оператор → MinIO admin | не опубликован, консоль выключена | — | root-ключи только у `minio-init` (bootstrap) | приложения используют только scoped-пользователей | — | аудит K (порт 9001 закрыт), C (у приложений нет createBucket) | +| Оператор → Prometheus/Loki/Tempo | `127.0.0.1:9090/3100/3200` | Root | `operator.messenger` (Telemetry Clients CA) | mTLS | 90 дн. | аудит F (operator → 200, без сертификата → отказ) | +| API → Valkey | `valkey:6379` TLS (plaintext port 0) | valkey: Root, SAN `valkey` | ACL-пользователь `api` (пароль из OpenBao) | ключи только `session:*` и `ratelimit:*`; команды get/set/del/incr/incrby/expire/ttl/ping; `default` выключен | пароль в OpenBao | аудит G: plaintext, чужой CA, без аутентификации, неверный пароль → отказ; другие ключи, FLUSHALL, CONFIG, KEYS → NOPERM | +| Replication / кластер | — | — | — | N/A: по одному узлу | — | аудит K: 25672/4369 (RabbitMQ), 8201 (OpenBao) закрыты | + +## Служебные каналы (scrape / health / bootstrap) + +| Канал | Где | Почему это не обход | +| --- | --- | --- | +| health API | `127.0.0.1:8081` внутри контейнера API, только `/health/live` и `/health/ready` | loopback-only; бизнес-API на нём нет (аудит K: 8081 закрыт из сети) | +| health nginx → API | `https://api-N:8080/health/ready` с LB-сертификатом | тот же mTLS, что у бизнес-трафика | +| self-метрики Collector | `127.0.0.1:8888`, сбор receiver `prometheus/self` внутри того же процесса | loopback-only; дальше идёт по mTLS-pipeline | +| Prometheus scrape | нет ни одного scrape-задания | все метрики приходят OTLP push по mTLS | +| healthcheck Loki/Tempo | TCP-connect на 127.0.0.1 | HTTP без клиентского сертификата невозможен | +| PostgreSQL local socket | `trust` только для Unix-сокета внутри контейнера | доступен только через `docker compose exec`, то есть оператору | +| Erlang distribution | `rabbit@localhost`, epmd и 25672 на 127.0.0.1 | из сети закрыто (аудит K) | +| bootstrap | `migrate`, `minio-init` — одноразовые задания с операторскими credentials | завершаются после настройки; у сервисов этих credentials нет | + +## Ограничения модели + +- Collector не связывает resource-атрибуты (`service.name`) с identity сертификата: `api-otel` может отправить данные с `service.name=messenger-worker`. Защищён сам факт ingest (только Ingest CA), но не подделка источника. +- В Prometheus, Loki и Tempo нет разграничения прав по identity: любой клиент Telemetry Clients CA и пишет, и читает. Least privilege обеспечен выдачей этого CA только трём identity (collector, grafana, operator). +- Отзыв клиентских сертификатов проверяется только в API (serial denylist). RabbitMQ, OpenBao и Collector отзыв не проверяют. Отозвать identity в RabbitMQ — удалить пользователя из `definitions.json`; в OpenBao — удалить роль `auth/cert/certs/`. +- Уже открытые соединения повторно не проверяются: отзыв действует на новые соединения (keep-alive nginx → API — 4 с). diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 0000000..c02e4d8 --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,244 @@ +# Панели dashboard «Messenger — наблюдаемость (lab6)» + +Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. `$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval). + +## SLO (окно 30 мин) и алерты + +**Доступность (SLO 99%)** (stat, единица: доля (0..1)) +— 1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы. + +```promql +messenger:slo_availability:ratio30m +``` + +**Латентность ≤500 ms (SLO 95%)** (stat, единица: доля (0..1)) +— Доля запросов в bucket le=0.5 s за 30 мин. + +```promql +messenger:slo_latency:ratio30m +``` + +**Error ratio 5xx (2m)** (stat, единица: доля (0..1)) + +```promql +messenger:http_error_ratio:rate2m +``` + +**Старейшее задание** (stat, единица: секунды) + +```promql +max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) +``` + +**Алерты Prometheus (firing/pending)** (alertlist) + +Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`). + + +**RPS API** (stat, единица: запросы/с) + +```promql +messenger:http_requests:rate2m +``` + +**p95 API (5m)** (stat, единица: секунды) + +```promql +messenger:http_latency_p95:rate5m +``` + +**Загрузка worker** (stat, единица: доля (0..1)) + +```promql +messenger:worker_saturation:ratio +``` + +**Ожидают пул БД** (stat, единица: штуки) + +```promql +max(db_client_connection_pending_requests) +``` + +## API (messenger-api, все реплики) + +**RPS по маршруту и методу** (timeseries, единица: запросы/с) +— Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID. + +```promql +sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # {{http_request_method}} {{http_route}} +``` + +**RPS по классу статуса** (timeseries, единица: запросы/с) + +```promql +sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d)..")) # {{status_class}} +``` + +**Латентность API p50/p95/p99 (из histogram buckets)** (timeseries, единица: секунды) +— histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик. + +```promql +histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p50 +``` +```promql +histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p95 +``` +```promql +histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p99 +``` + +**Доля ошибок 5xx** (timeseries, единица: доля (0..1)) +— Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом. + +```promql +(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # 5xx / все +``` + +**p95 по маршруту** (timeseries, единица: секунды) + +```promql +histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # {{http_route}} +``` + +**Запросы в обработке** (timeseries, единица: штуки) + +```promql +sum by (instance) (http_server_active_requests{service_name="messenger-api"}) # {{instance}} +``` + +## Очередь и worker + +**Jobs по исходу (в сек.)** (timeseries, единица: операции/с) +— succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1 + +```promql +sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval])) # {{outcome}} +``` + +**Длительность job (succeeded) p50/p95/p99** (timeseries, единица: секунды) + +```promql +histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p50 +``` +```promql +histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p95 +``` +```promql +histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p99 +``` + +**Глубина очереди (из PostgreSQL)** (timeseries, единица: штуки) +— max() по репликам API: каждая публикует одно и то же значение из БД. + +```promql +max by (queue, state) (messenger_queue_depth) # {{queue}} {{state}} +``` + +**Возраст старейшей job** (timeseries, единица: секунды) +— От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency. + +```promql +max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) # oldest age +``` + +**Время до ready p50/p95/p99** (timeseries, единица: секунды) +— messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready. + +```promql +histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p50 +``` +```promql +histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p95 +``` +```promql +histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p99 +``` + +**Насыщение worker: занято / ёмкость** (timeseries, единица: штуки) + +```promql +sum(messenger_worker_busy) # busy +``` +```promql +sum(messenger_worker_capacity) # capacity +``` +```promql +messenger:worker_saturation:ratio # saturation +``` + +## Ресурсы: пул БД, CPU, RAM + +**Пул БД: used / idle / waiting** (timeseries, единица: штуки) + +```promql +sum by (service_name, db_client_connection_state) (db_client_connection_count) # {{service_name}} {{db_client_connection_state}} +``` +```promql +sum by (service_name) (db_client_connection_pending_requests) # {{service_name}} waiting +``` +```promql +sum by (service_name) (db_client_connection_max) # {{service_name}} max +``` + +**CPU процессов (ядра)** (timeseries, единица: доля (0..1)) +— worker: self + завершённые дочерние процессы обработки изображений. + +```promql +sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval])) # {{service_name}} {{instance}} +``` +```promql +rate(otelcol_process_cpu_seconds_total[$__rate_interval]) # otel-collector +``` + +**RSS процессов** (timeseries, единица: байты) + +```promql +process_memory_usage_bytes # {{service_name}} {{instance}} +``` +```promql +otelcol_process_memory_rss_bytes # otel-collector +``` + +**Telemetry pipeline: принято / отправлено / ошибки (items/s)** (timeseries, единица: штуки) + +```promql +sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval])) # accepted spans +``` +```promql +sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval])) # accepted log records +``` +```promql +sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval])) # accepted metric points +``` +```promql +sum by (exporter) (rate({__name__=~"otelcol_exporter_sent_(spans|log_records|metric_points)_total"}[$__rate_interval])) # sent {{exporter}} +``` +```promql +sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0) # send failed spans +``` +```promql +sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0) # dropped spans (queue full) +``` +```promql +sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0) # send failed logs +``` +```promql +sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0) # dropped logs (queue full) +``` +```promql +sum by (exporter) (otelcol_exporter_queue_size) # queue {{exporter}} +``` + +## Логи и трассы + +**Логи API и worker (warning/error)** (logs) + +``` +{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR" +``` + +**Трассы с ошибками или медленнее 500 ms (TraceQL)** (table) + +``` +{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)} +``` diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..d0fa9ec --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,90 @@ +# Runbooks алертов (lab6) + +Алерты вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`) каждые 15 с. Состояние смотрится в Grafana +(dashboard «Messenger — наблюдаемость», панель «Алерты Prometheus») или командой: + +```sh +curl -s http://127.0.0.1:9090/api/v1/alerts | python3 -m json.tool +``` + +Доставка во внешние каналы не настроена (по заданию достаточно локального firing/resolved). + +Общий порядок диагностики: **метрика → трасса → лог**. Метрика говорит «что и сколько», трасса — «где в +цепочке тратится время / где ошибка», лог — «какое событие и с какими техническими ID». +Все шаги ниже используют только сохранённую telemetry. + +--- + +## MessengerApiHighErrorRate + +**Условие:** `messenger:http_error_ratio:rate2m > 0.05` при `messenger:http_requests:rate2m > 0.2`, `for: 1m`. +Числитель — ответы 5xx, знаменатель — все бизнес-запросы messenger-api (health-пробы не инструментированы). +4xx (неверный пароль, чужой чат, 409) в числитель не входят. + +1. Dashboard → «RPS по классу статуса» и «Доля ошибок 5xx»: какой код (500 или 503) и с какого момента. +2. Prometheus: на каком маршруте ошибки + `sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))`. +3. Отличить БД от handler: + - пул: `max(db_client_connection_pending_requests)`, `db_client_connection_count{db_client_connection_state="used"}`. + Ожидание пула > 0 → соединения заняты долгими запросами к БД; + - Tempo (Explore → Tempo → TraceQL): + `{resource.service.name="messenger-api" && span.db.system="postgresql" && status=error}` — если в трассе + db-span (`INSERT`/`SELECT`) занимает почти всё время запроса и завершился исключением + `LockNotAvailable`/`QueryCanceled`/`OperationalError` — причина в PostgreSQL (блокировка, перегрузка, недоступность); + - если server span длинный, а db-spans короткие (единицы ms) — медленный/падающий handler + (`{span.messenger.fault="slow"}` в учебном стенде даёт именно такую картину). +4. Loki: `{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"` — события `unhandled_error` + (500, есть stack trace в атрибуте), `not_ready` (БД недоступна), `request_timeout`. + По `X-Request-Id`, который получил клиент: `{service_name="messenger-api"} |= ""` → поле `trace_id` + → кнопка «Open trace». +5. Если виновата БД: `docker compose exec postgres psql -U messenger -d messenger -c "SELECT pid, state, + wait_event_type, now()-xact_start AS age, left(query,80) FROM pg_stat_activity WHERE state <> 'idle'"` — + найти блокирующую транзакцию; согласовать и выполнить `SELECT pg_terminate_backend()`. +6. Восстановление: доля 5xx падает сразу, alert уходит в resolved, когда окно 2m очистится (~2–3 мин). + +## MessengerQueueLagGrowing + +**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60`, `for: 1m`. +Возраст считается от COMMIT задания (момент ответа 202) по данным PostgreSQL; публикуют все реплики API, +поэтому `max()`. + +1. Dashboard → «Глубина очереди», «Jobs по исходу», «Насыщение worker». + - `sum(rate(messenger_jobs_processed_total[1m])) == 0` при растущей глубине → worker не работает; + - `messenger:worker_saturation:ratio ≈ 1` и jobs/s > 0 → worker не успевает (нужно больше реплик/конкурентности); + - много `outcome="retry"` → ошибки зависимостей (S3/DB), см. логи `job_attempt_failed`. +2. Tempo: найти upload по attachment_id `{span.messenger.attachment_id=""}`. Если в трассе есть + PRODUCER `attachments.process publish`, но нет CONSUMER — сообщение в брокере, worker его не забрал. + Если нет и PRODUCER — outbox не опубликован: смотреть `max(messenger_queue_depth{queue="outbox"})` и лог + `outbox_relay_paused` (брокер недоступен). +3. Loki: `{service_name="messenger-worker"}` — последние события `worker_stopping`/`worker_stopped` или отсутствие + `job_*` событий; `{service_name="messenger-worker"} | event="job_attempt_failed"` — причины retry. +4. Действие: `docker compose ps worker`, `docker compose start worker` (или масштабирование + `docker compose up -d --scale worker=3`). Задания не теряются: они в PostgreSQL/outbox и в durable очереди. +5. Восстановление: backlog уходит за секунды, возраст падает до 0, alert resolved через ~15–30 с. + +## MessengerApiLatencyHigh + +**Условие:** p95 из buckets всех реплик за 5m > 1 s, `for: 2m`. Шаги — как в HighErrorRate, пункт 3: +по трассам `{resource.service.name="messenger-api" && duration > 1s}` определить, какой span занимает время +(db / S3 / handler). + +## MessengerDbPoolWaiting + +**Условие:** `max(db_client_connection_pending_requests) > 0`, `for: 1m`. Пул исчерпан: долгие запросы или +блокировки в БД (см. HighErrorRate, п. 3 и 5) либо пул мал для нагрузки (`DB_POOL_MAX`). + +## MessengerWorkerSaturated + +**Условие:** занято > 90% слотов всех worker 5 минут. Если очередь при этом растёт — масштабировать worker; +проверить `messenger_job_duration_seconds` p95 (не стали ли задания медленнее). + +## TelemetryExportFailing + +**Условие:** очередь exporter заполнена больше чем на 20% либо есть send_failed/enqueue_failed > 0, `for: 1m`. +Бизнес-API **не** затронут: SDK и collector имеют ограниченные очереди. + +1. `docker compose ps tempo loki prometheus` — какой backend недоступен. +2. Панель «Telemetry pipeline»: `otelcol_exporter_queue_size` по exporter (ёмкость 100 батчей) и + `otelcol_exporter_enqueue_failed_*` — сколько данных уже отброшено. +3. `docker compose start tempo loki`. Данные, отброшенные при переполнении очереди, не восстанавливаются — + их объём виден в счётчиках и фиксируется в отчёте инцидента. diff --git a/evidence/auth_acl.txt b/evidence/auth_acl.txt new file mode 100644 index 0000000..19934a9 --- /dev/null +++ b/evidence/auth_acl.txt @@ -0,0 +1,11 @@ +10:06:50 PASS acl_outsider eve: 404 on chat/history/metadata/content/variant/post, cannot attach a foreign file +10:06:50 PASS identity_spoofing forged X-User-Id ignored; without Bearer 401; Basic on business endpoint 401 +10:06:50 PASS logout revoked token: 401 on ['api-1', 'api-2'] +10:06:50 session expires_at moved to the past in PostgreSQL; the Valkey cache entry lives at most 30 s +10:07:22 PASS expiry expired session: 401 on ['api-1', 'api-2'] after the 30 s cache bound +10:07:23 PASS rehash stored m=8192,t=1,p=1 -> m=19456,t=2,p=1 after a successful login +10:07:23 PASS size_limits JSON 70 kB -> 413 (payload_too_large), upload > 10 MiB -> 413 +10:07:23 PASS rate_limit_user 5 failures on ['api-1', 'api-2'] -> 429 Retry-After=60s even with the right password +10:07:24 PASS rate_limit_ip per-IP limit: 429 after 107 attempts from one address +10:07:24 waiting for the rate-limit window to expire so that later tests are not affected +PASS: acl_outsider, identity_spoofing, logout, expiry, rehash, size_limits, rate_limit_user, rate_limit_ip diff --git a/evidence/bao_sealed.txt b/evidence/bao_sealed.txt new file mode 100644 index 0000000..b1bf17f --- /dev/null +++ b/evidence/bao_sealed.txt @@ -0,0 +1,8 @@ +11:08:38 openbao: sealed +11:08:40 sealed: running services keep working -> PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +11:09:00 refresher: openbao_unavailable ... keeping cached secret version (seen in API/worker logs) +11:09:35 worker-2 restarted while sealed: health=starting, waiting for OpenBao=True +11:09:36 upload during the seal processed by the remaining worker: ready +11:09:37 openbao: unsealed with 2 of 3 operator shares +11:09:52 worker-2 healthy 16 s after unseal (sealed for 58 s in total) +PASS: sealed OpenBao -> running services continue, new process waits, recovery after unseal diff --git a/evidence/cert_rotation.txt b/evidence/cert_rotation.txt new file mode 100644 index 0000000..5fabd99 --- /dev/null +++ b/evidence/cert_rotation.txt @@ -0,0 +1,11 @@ +10:12:23 current lb-client serial 0E30F0061EA3866362334D7C34B7F93B9BD61418 +10:12:24 issued lb-client (lb.client.messenger) serial=0E30F0061EA3866362334D7C34B7F93B9BD6142A valid 90d +10:12:25 nginx reloaded with the new client certificate (1.4 s after issue) +10:12:25 before revocation: old=HTTP/1.1 200 OK new=HTTP/1.1 200 OK +10:12:25 revoked serial=0E30F0061EA3866362334D7C34B7F93B9BD61418 +10:12:28 api-1 after revocation: old -> closed without response | new -> HTTP/1.1 200 OK | expired -> closed without response +10:12:29 api-2 after revocation: old -> closed without response | new -> HTTP/1.1 200 OK | expired -> closed without response +10:12:29 API log: mtls_client_rejected reason=revoked serial=0E30F0061EA3866362334D7C34B7F93B9BD61418 +10:12:34 traffic during rotation: 161 requests, 161 ok, unexpected errors 0 +10:12:35 PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +PASS: lb-client rotated (0E30F0061EA3866362334D7C34B7F93B9BD61418 -> 0E30F0061EA3866362334D7C34B7F93B9BD6142A) without image rebuild, old serial revoked, expired cert refused diff --git a/evidence/dashboard_check.txt b/evidence/dashboard_check.txt new file mode 100644 index 0000000..4d3a20d --- /dev/null +++ b/evidence/dashboard_check.txt @@ -0,0 +1,92 @@ +ok 1 series Доступность (SLO 99%) Доступность (SLO 99%) +ok 1 series Латентность ≤500 ms (SLO 95%) Латентность ≤500 ms (SLO 95%) +ok 1 series Error ratio 5xx (2m) Error ratio 5xx (2m) +ok 1 series Старейшее задание Старейшее задание +ok 1 series RPS API RPS API +ok 1 series p95 API (5m) p95 API (5m) +ok 1 series Загрузка worker Загрузка worker +ok 1 series Ожидают пул БД Ожидают пул БД +ok 12 series RPS по маршруту и методу {{http_request_method}} {{http_route}} +ok 3 series RPS по классу статуса {{status_class}} +ok 1 series Латентность API p50/p95/p99 (из histogram bucket p50 +ok 1 series Латентность API p50/p95/p99 (из histogram bucket p95 +ok 1 series Латентность API p50/p95/p99 (из histogram bucket p99 +ok 1 series Доля ошибок 5xx 5xx / все +ok 10 series p95 по маршруту {{http_route}} +ok 2 series Запросы в обработке {{instance}} +ok 6 series Jobs по исходу (в сек.) {{outcome}} +ok 1 series Длительность job (succeeded) p50/p95/p99 p50 +ok 1 series Длительность job (succeeded) p50/p95/p99 p95 +ok 1 series Длительность job (succeeded) p50/p95/p99 p99 +ok 3 series Глубина очереди (из PostgreSQL) {{queue}} {{state}} +ok 1 series Возраст старейшей job oldest age +ok 1 series Время до ready p50/p95/p99 p50 +ok 1 series Время до ready p50/p95/p99 p95 +ok 1 series Время до ready p50/p95/p99 p99 +ok 1 series Насыщение worker: занято / ёмкость busy +ok 1 series Насыщение worker: занято / ёмкость capacity +ok 1 series Насыщение worker: занято / ёмкость saturation +ok 4 series Пул БД: used / idle / waiting {{service_name}} {{db_client_connection_state}} +ok 2 series Пул БД: used / idle / waiting {{service_name}} waiting +ok 2 series Пул БД: used / idle / waiting {{service_name}} max +ok 4 series CPU процессов (ядра) {{service_name}} {{instance}} +ok 1 series CPU процессов (ядра) otel-collector +ok 4 series RSS процессов {{service_name}} {{instance}} +ok 1 series RSS процессов otel-collector +ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted spans +ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted log records +ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted metric points +ok 3 series Telemetry pipeline: принято / отправлено / ошибк sent {{exporter}} +ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed spans +ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped spans (queue full) +ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed logs +ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped logs (queue full) +ok 3 series Telemetry pipeline: принято / отправлено / ошибк queue {{exporter}} +panels: 32, prometheus queries without data: 0 +grafana prometheus ok 84 values Доступность (SLO 99%) +grafana prometheus ok 84 values Латентность ≤500 ms (SLO 95%) +grafana prometheus ok 84 values Error ratio 5xx (2m) +grafana prometheus ok 86 values Старейшее задание +grafana prometheus ok 84 values RPS API +grafana prometheus ok 84 values p95 API (5m) +grafana prometheus ok 86 values Загрузка worker +grafana prometheus ok 86 values Ожидают пул БД +grafana prometheus ok 789 values RPS по маршруту и методу +grafana prometheus ok 179 values RPS по классу статуса +grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket +grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket +grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket +grafana prometheus ok 82 values Доля ошибок 5xx +grafana prometheus ok 683 values p95 по маршруту +grafana prometheus ok 170 values Запросы в обработке +grafana prometheus ok 273 values Jobs по исходу (в сек.) +grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99 +grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99 +grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99 +grafana prometheus ok 258 values Глубина очереди (из PostgreSQL) +grafana prometheus ok 86 values Возраст старейшей job +grafana prometheus ok 77 values Время до ready p50/p95/p99 +grafana prometheus ok 77 values Время до ready p50/p95/p99 +grafana prometheus ok 77 values Время до ready p50/p95/p99 +grafana prometheus ok 86 values Насыщение worker: занято / ёмкость +grafana prometheus ok 86 values Насыщение worker: занято / ёмкость +grafana prometheus ok 86 values Насыщение worker: занято / ёмкость +grafana prometheus ok 344 values Пул БД: used / idle / waiting +grafana prometheus ok 172 values Пул БД: used / idle / waiting +grafana prometheus ok 172 values Пул БД: used / idle / waiting +grafana prometheus ok 350 values CPU процессов (ядра) +grafana prometheus ok 91 values CPU процессов (ядра) +grafana prometheus ok 402 values RSS процессов +grafana prometheus ok 105 values RSS процессов +grafana prometheus ok 76 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 80 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 247 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 273 values Telemetry pipeline: принято / отправлено / ошибк +grafana loki ok 100 values Логи API и worker (warning/error) +grafana tempo ok 20 traces Трассы с ошибками или медленнее 500 ms (TraceQL) (datasource proxy) +via Grafana: queries without data or with errors: 0 diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..38cb417 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,13 @@ +# 2026-11-11T08:10:20Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 1) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.265} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab7-api-1-1 Up 38 seconds (healthy) +$ docker compose start api-1 api-2 +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 2) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.672} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab7-api-1-1 Up About a minute (healthy) +$ docker compose start api-1 api-2 diff --git a/evidence/hardening.json b/evidence/hardening.json new file mode 100644 index 0000000..f724860 --- /dev/null +++ b/evidence/hardening.json @@ -0,0 +1,342 @@ +[ + { + "container": "api-1-1", + "user": "10001:10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 0.5, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "api-2-1", + "user": "10001:10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 0.5, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "grafana-1", + "user": "472", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 0.5, + "memory_mib": 256, + "pids_limit": 256, + "docker_socket": false, + "published": [ + "127.0.0.1:3000->3000/tcp" + ], + "problems": [] + }, + { + "container": "loki-1", + "user": "10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [], + "cpus": 1.0, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [ + "127.0.0.1:3100->3100/tcp" + ], + "problems": [] + }, + { + "container": "minio-1", + "user": "65532", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 0.5, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "nginx-1", + "user": "101:101", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/etc/nginx/upstream", + "/tmp", + "/var/cache/nginx", + "/var/run" + ], + "cpus": 0.25, + "memory_mib": 64, + "pids_limit": 256, + "docker_socket": false, + "published": [ + "127.0.0.1:8443->8443/tcp" + ], + "problems": [] + }, + { + "container": "openbao-1", + "user": "100:1000", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 0.5, + "memory_mib": 256, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "otel-collector-1", + "user": "10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [], + "cpus": 1.0, + "memory_mib": 256, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "postgres-1", + "user": "70:70", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp", + "/var/run/postgresql" + ], + "cpus": 1.0, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "prometheus-1", + "user": "nobody", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [], + "cpus": 1.0, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [ + "127.0.0.1:9090->9090/tcp" + ], + "problems": [] + }, + { + "container": "rabbitmq-1", + "user": "100:101", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/etc/rabbitmq/conf.d", + "/tmp" + ], + "cpus": 1.0, + "memory_mib": 512, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "tempo-1", + "user": "10001:10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 1.0, + "memory_mib": 1024, + "pids_limit": 256, + "docker_socket": false, + "published": [ + "127.0.0.1:3200->3200/tcp" + ], + "problems": [] + }, + { + "container": "valkey-1", + "user": "999:1000", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/data", + "/tmp" + ], + "cpus": 0.25, + "memory_mib": 128, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "worker-1", + "user": "10001:10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 2.0, + "memory_mib": 1536, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + }, + { + "container": "worker-2", + "user": "10001:10001", + "privileged": false, + "network": "mtusi-lab7_default", + "pid": "private", + "cap_drop": [ + "ALL" + ], + "cap_add": null, + "no_new_privileges": true, + "read_only_rootfs": true, + "tmpfs": [ + "/tmp" + ], + "cpus": 2.0, + "memory_mib": 1536, + "pids_limit": 256, + "docker_socket": false, + "published": [], + "problems": [] + } +] \ No newline at end of file diff --git a/evidence/hardening.txt b/evidence/hardening.txt new file mode 100644 index 0000000..d77d9df --- /dev/null +++ b/evidence/hardening.txt @@ -0,0 +1,18 @@ +container user capDrop nnp ro cpus mem/pids published problems +api-1-1 10001:10001 ALL True True 0.5 512/256 - ok +api-2-1 10001:10001 ALL True True 0.5 512/256 - ok +grafana-1 472 ALL True True 0.5 256/256 127.0.0.1:3000 ok +loki-1 10001 ALL True True 1.0 512/256 127.0.0.1:3100 ok +minio-1 65532 ALL True True 0.5 512/256 - ok +nginx-1 101:101 ALL True True 0.25 64/256 127.0.0.1:8443 ok +openbao-1 100:1000 ALL True True 0.5 256/256 - ok +otel-collector-1 10001 ALL True True 1.0 256/256 - ok +postgres-1 70:70 ALL True True 1.0 512/256 - ok +prometheus-1 nobody ALL True True 1.0 512/256 127.0.0.1:9090 ok +rabbitmq-1 100:101 ALL True True 1.0 512/256 - ok +tempo-1 10001:10001 ALL True True 1.0 1024/256 127.0.0.1:3200 ok +valkey-1 999:1000 ALL True True 0.25 128/256 - ok +worker-1 10001:10001 ALL True True 2.0 1536/256 - ok +worker-2 10001:10001 ALL True True 2.0 1536/256 - ok +published ports allowed: {'8443': 'nginx business ingress (TLS)', '3000': 'Grafana (HTTPS + login)', '9090': 'Prometheus (mTLS operator)', '3100': 'Loki (mTLS operator)', '3200': 'Tempo (mTLS operator)'} +15 containers, 0 problems diff --git a/evidence/image_scan.txt b/evidence/image_scan.txt new file mode 100644 index 0000000..5d080cf --- /dev/null +++ b/evidence/image_scan.txt @@ -0,0 +1,16 @@ +Trivy 0.58.1, DB 2026-11-11; scanned with docker save (no Docker socket in the scanner) + +== app BEFORE (python 3.12.8, pillow 11.0.0, urllib3 2.3.0) + /scan/app.tar (debian 12.9) os-pkgs {'CRITICAL': 9, 'HIGH': 145, 'LOW': 158, 'MEDIUM': 217} | HIGH+CRIT with fix: 56/154 | secrets: 0 + Python lang-pkgs {'CRITICAL': 5, 'HIGH': 43, 'LOW': 2, 'MEDIUM': 26} | HIGH+CRIT with fix: 48/48 | secrets: 0 + +== app AFTER (python 3.12.16, pillow 13.0.0, urllib3 2.8.0) + /scan/app.tar (debian 12.15) os-pkgs {'CRITICAL': 2, 'HIGH': 96, 'LOW': 108, 'MEDIUM': 129, 'UNKNOWN': 10} | HIGH+CRIT with fix: 0/98 | secrets: 0 + Python lang-pkgs {'HIGH': 3, 'LOW': 2, 'MEDIUM': 9} | HIGH+CRIT with fix: 3/3 | secrets: 0 + +== nginx:1.27.3-alpine + /scan/nginx.tar (alpine 3.20.5) os-pkgs {'CRITICAL': 3, 'HIGH': 37, 'LOW': 31, 'MEDIUM': 33} | HIGH+CRIT with fix: 40/40 | secrets: 0 + +== openbao/openbao:2.1.0 + /scan/openbao.tar (alpine 3.20.3) os-pkgs {'CRITICAL': 2, 'HIGH': 26, 'LOW': 23, 'MEDIUM': 14} | HIGH+CRIT with fix: 28/28 | secrets: 0 + bin/bao lang-pkgs {'CRITICAL': 2, 'HIGH': 61, 'LOW': 9, 'MEDIUM': 68, 'UNKNOWN': 2} | HIGH+CRIT with fix: 59/63 | secrets: 0 diff --git a/evidence/lab5_failure_windows.txt b/evidence/lab5_failure_windows.txt new file mode 100644 index 0000000..9b8728c --- /dev/null +++ b/evidence/lab5_failure_windows.txt @@ -0,0 +1,80 @@ +10:18:35 scenario A: scenario_a +10:18:35 client saw: missing X-Request-Id +10:18:35 committed attachment a3900cbb-c50d-496c-9962-ba0ec39928d3 survived the API crash +10:18:35 statuses: processing -> ready +10:18:35 audit attempt=1 enqueued api:api-1 +10:18:35 audit attempt=1 claimed 2e84bfa8176e:4f81b283 +10:18:35 audit attempt=1 succeeded 2e84bfa8176e:4f81b283 +10:18:36 scenario A PASS (1.9s) +10:18:36 scenario B: scenario_b +10:18:59 statuses: queued -> processing -> queued -> ready +10:18:59 audit attempt=1 enqueued api:api-2 +10:18:59 audit attempt=1 claimed eb9974d4233a:3bafe435 +10:18:59 audit attempt=1 retry_scheduled sweeper:2e84bfa8176e:4f81b28 lease_expired delay=1.34s +10:18:59 audit attempt=2 claimed 2e84bfa8176e:4f81b283 +10:18:59 audit attempt=2 succeeded 2e84bfa8176e:4f81b283 +10:19:00 scenario B PASS (23.6s) +10:19:00 scenario C: scenario_c +10:19:01 statuses: queued -> ready +10:19:01 audit attempt=1 enqueued api:api-2 +10:19:01 audit attempt=1 claimed eb9974d4233a:f07c7c15 +10:19:01 audit attempt=1 succeeded eb9974d4233a:f07c7c15 +10:19:11 scenario C PASS (10.8s) +10:19:11 scenario D: scenario_d +10:19:22 statuses: processing -> queued -> ready +10:19:22 audit attempt=1 enqueued api:api-1 +10:19:22 audit attempt=1 claimed 2e84bfa8176e:4f81b283 +10:19:22 audit attempt=1 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=1.33s +10:19:22 audit attempt=2 claimed eb9974d4233a:ea4f119d +10:19:22 audit attempt=2 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=2.5s +10:19:22 audit attempt=3 claimed 2e84bfa8176e:4f81b283 +10:19:22 audit attempt=3 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=5.78s +10:19:22 audit attempt=4 claimed eb9974d4233a:ea4f119d +10:19:22 audit attempt=4 succeeded eb9974d4233a:ea4f119d +10:19:24 backoff delays: [1.33, 2.5, 5.78] +10:19:24 scenario D PASS (12.6s) +10:19:24 scenario E: scenario_e +10:19:24 audit attempt=1 enqueued api:api-2 +10:19:24 audit attempt=1 claimed 2e84bfa8176e:4f81b283 +10:19:24 audit attempt=1 failed 2e84bfa8176e:4f81b283 invalid_image +10:19:24 scenario E PASS (0.7s) +10:19:24 scenario F: scenario_f +10:19:48 failed registry contains the job; replay +10:20:11 statuses: queued -> ready +10:20:11 audit attempt=1 enqueued api:api-1 +10:20:11 audit attempt=1 claimed eb9974d4233a:ea4f119d +10:20:11 audit attempt=1 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=1.28s +10:20:11 audit attempt=2 claimed 2e84bfa8176e:4f81b283 +10:20:11 audit attempt=2 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=2.55s +10:20:11 audit attempt=3 claimed eb9974d4233a:ea4f119d +10:20:11 audit attempt=3 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=6.47s +10:20:11 audit attempt=4 claimed 2e84bfa8176e:4f81b283 +10:20:11 audit attempt=4 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=9.94s +10:20:11 audit attempt=5 claimed eb9974d4233a:ea4f119d +10:20:11 audit attempt=5 failed eb9974d4233a:ea4f119d retry_exhausted +10:20:11 audit attempt=6 replayed operator:test-suite +10:20:11 audit attempt=6 claimed 2e84bfa8176e:4f81b283 +10:20:11 audit attempt=6 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=20.59s +10:20:11 audit attempt=7 claimed eb9974d4233a:ea4f119d +10:20:11 audit attempt=7 succeeded eb9974d4233a:ea4f119d +10:20:13 second replay refused: job is succeeded, attachment is ready +10:20:14 scenario F PASS (50.2s) +10:20:14 scenario G: scenario_g +10:20:38 statuses: processing -> queued -> processing -> ready +10:20:38 audit attempt=1 enqueued api:api-1 +10:20:38 audit attempt=1 claimed 2e84bfa8176e:4f81b283 +10:20:38 audit attempt=1 retry_scheduled sweeper:eb9974d4233a:ea4f119 lease_expired delay=1.12s +10:20:38 audit attempt=2 claimed eb9974d4233a:ea4f119d +10:20:38 audit attempt=2 succeeded eb9974d4233a:ea4f119d +10:21:11 scenario G PASS (56.4s) +10:21:11 scenario H: scenario_h +10:21:15 DLQ depth 0 -> 1 +10:21:15 scenario H PASS (4.1s) +10:21:15 scenario I: scenario_i +10:21:19 broker down: upload 202, outbox holds 1 unpublished job +10:21:25 statuses: queued -> ready +10:21:25 audit attempt=1 enqueued api:api-1 +10:21:25 audit attempt=1 claimed 2e84bfa8176e:4f81b283 +10:21:25 audit attempt=1 succeeded 2e84bfa8176e:4f81b283 +10:21:26 scenario I PASS (11.4s) +PASS: failure windows A B C D E F G H I diff --git a/evidence/lab6_worker_down.txt b/evidence/lab6_worker_down.txt new file mode 100644 index 0000000..eaa5d30 --- /dev/null +++ b/evidence/lab6_worker_down.txt @@ -0,0 +1,14 @@ +10:18:08 stop all workers +10:18:08 upload 202 queued id=59f4941d-1a35-4d70-a976-c8ed89746f38 +10:18:08 message 0f104c92-c0dd-46a0-b906-f58c9fde4017 created with queued attachment +10:18:10 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0} +10:18:11 restart api-1 api-2 rabbitmq (volumes kept) +10:18:23 started before: mtusi-lab7-api-1-1 2026-11-11T07:17:26.446583425Z; mtusi-lab7-api-2-1 2026-11-11T07:17:38.07347875Z; mtusi-lab7-rabbitmq-1 2026-11-11T07:17:27.208089551Z +10:18:23 started after: mtusi-lab7-api-1-1 2026-11-11T07:18:11.610777668Z; mtusi-lab7-api-2-1 2026-11-11T07:18:11.669354168Z; mtusi-lab7-rabbitmq-1 2026-11-11T07:18:12.345023127Z +10:18:24 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0} +10:18:24 start workers +10:18:27 status sequence observed: queued -> ready +10:18:28 objects: ['attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/original', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/crop.jpg', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/optimized.webp', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/thumbnail.jpg'] +10:18:28 redelivery: republish the finished job through the outbox (duplicate message) +10:18:34 duplicate acked as no-op: variants=3 successes=1 +PASS: worker down / API+broker restart / redelivery no-op diff --git a/evidence/login_budget.json b/evidence/login_budget.json new file mode 100644 index 0000000..85cc4da --- /dev/null +++ b/evidence/login_budget.json @@ -0,0 +1,87 @@ +{ + "argon2_bench_api_container_0_5_cpu": [ + { + "m_kib": 19456, + "t": 2, + "p": 1, + "p50_ms": 11.3, + "max_ms": 66.6, + "peak_rss_mib": 37.3 + }, + { + "m_kib": 12288, + "t": 3, + "p": 1, + "p50_ms": 9.3, + "max_ms": 62.0, + "peak_rss_mib": 37.3 + }, + { + "m_kib": 47104, + "t": 1, + "p": 1, + "p50_ms": 16.6, + "max_ms": 69.2, + "peak_rss_mib": 83.0 + }, + { + "m_kib": 65536, + "t": 3, + "p": 1, + "p50_ms": 171.6, + "max_ms": 175.5, + "peak_rss_mib": 101.0 + }, + { + "m_kib": 65536, + "t": 3, + "p": 4, + "p50_ms": 390.0, + "max_ms": 806.1, + "peak_rss_mib": 101.2 + } + ], + "baseline": { + "attackers": 0, + "seconds": 35, + "login_attempts_per_s": 0.0, + "login_status_counts": {}, + "login_401_p50_ms": null, + "login_401_p99_ms": null, + "probe_ok": 165, + "probe_errors": 0, + "probe_p50_ms": 9.5, + "probe_p99_ms": 46.4, + "api_cpu_pct_of_one_core_both_replicas": 10.3, + "api_rss_samples": { + "mtusi-lab7-api-2-1": "97.59MiB", + "mtusi-lab7-api-1-1": "97.34MiB" + } + }, + "flood": { + "attackers": 32, + "seconds": 37, + "login_attempts_per_s": 40.4, + "login_status_counts": { + "401": 1496 + }, + "login_401_p50_ms": 802.6, + "login_401_p99_ms": 2396.6, + "probe_ok": 128, + "probe_errors": 0, + "probe_p50_ms": 28.5, + "probe_p99_ms": 295.0, + "api_cpu_pct_of_one_core_both_replicas": 89.6, + "api_rss_samples": { + "mtusi-lab7-api-2-1": "196.7MiB", + "mtusi-lab7-api-1-1": "183.3MiB" + } + }, + "config": { + "PASSWORD_HASH_CONCURRENCY_per_replica": 4, + "PASSWORD_HASH_QUEUE_SECONDS": 3, + "replicas": 2, + "api_cpu_limit": 0.5, + "api_mem_limit_mib": 512 + } +} diff --git a/evidence/observability.txt b/evidence/observability.txt new file mode 100644 index 0000000..c817fac --- /dev/null +++ b/evidence/observability.txt @@ -0,0 +1,27 @@ +10:32:44 message sent request_id=94e97b26265542fb9d208a905b5d5701 (text marker kept private) +10:32:47 upload 913f8afd-4a07-4a58-b943-ea623e6ec0d0: queued -> queued -> processing -> ready +10:32:51 Tempo: trace 715b953a7f7536b44b99c48205a5c639, 42 spans; tree (DB client spans omitted): + api SERVER POST /api/v1/chats/{chat_id}/attachments 15.3 ms UNSET {"http.response.status_code": "202", "messenger.request_id": "95e2a7eb2f9e46319830eca2e087ccfb", "messenger.instance_id": "api-2", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"} + api CLIENT S3.PutObject 8.4 ms UNSET {"rpc.method": "PutObject"} + api PRODUCER attachments.process publish 3.1 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.job.attempt": "1", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"} + worker CONSUMER attachments.process process 8.3 ms ERROR {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0", "messenger.job.attempt": "1", "messenger.job.outcome": "retry"} + api PRODUCER attachments.process publish 4.8 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.job.attempt": "2", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"} + worker CONSUMER attachments.process process 174.7 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0", "messenger.job.attempt": "2", "messenger.job.outcome": "succeeded"} + worker CLIENT S3.GetObject 15.2 ms UNSET {"rpc.method": "GetObject"} + worker INTERNAL imageproc 128.0 ms UNSET {"messenger.pipeline_version": "1"} + worker CLIENT S3.CopyObject 5.3 ms UNSET {"rpc.method": "CopyObject"} + worker CLIENT S3.PutObject 3.0 ms UNSET {"rpc.method": "PutObject"} + worker CLIENT S3.PutObject 2.1 ms UNSET {"rpc.method": "PutObject"} + worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"} + worker CLIENT S3.DeleteObject 1.0 ms UNSET {"rpc.method": "DeleteObject"} +10:32:51 PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links +10:32:51 Loki: 4 records with trace_id=715b953a7f7536b44b99c48205a5c639: [('api', 'attachment_queued'), ('worker', 'job_claimed'), ('worker', 'job_attempt_failed'), ('worker', 'job_finished')] +10:32:51 Loki record fields: ['code_filepath', 'code_function', 'code_lineno', 'deployment_environment_name', 'detected_level', 'event', 'flags', 'observed_timestamp', 'scope_name', 'service_instance_id', 'service_name', 'service_namespace', 'service_version', 'severity_number', 'severity_text', 'span_id', 'telemetry_sdk_language', 'telemetry_sdk_name', 'telemetry_sdk_version', 'trace_id', 'ts'] +10:32:55 HTTP 500 X-Request-Id=2df37bc19c3d4676804688b4abcb2ae1 -> Loki event=unhandled_error trace_id=f63428fc5f86c4f25be261c5db8ecc08 +10:32:55 Tempo: GET /api/v1/chats status=ERROR request_id matches; exception event recorded=True +10:32:55 message POST trace 3157b85474b226259a91fb7e6ddeec5a: ['POST /api/v1/chats/{chat_id}/messages', 'SELECT', 'SELECT chat_members', 'SELECT idempotency_keys', 'INSERT messages', 'INSERT idempotency_keys'] +10:32:55 PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from traces and logs +10:32:55 PASS: 1921 series, label names: ['__name__', 'db_client_connection_state', 'decision', 'deployment_environment_name', 'error_type', 'http_method', 'http_request_method', 'http_response_status_code', 'http_route', 'http_scheme', 'http_status_code', 'instance', 'job', 'job_type', 'le', 'net_host_name', 'net_host_port', 'net_protocol_name', 'net_protocol_version', 'network_protocol_version', 'otel_scope_name', 'otel_scope_version', 'outcome', 'process_cpu_owner', 'queue', 'rule', 'service_instance_id', 'service_name', 'service_version', 'state', 'url_scheme'] +10:32:55 http_route values (templates only): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me'] +10:33:07 PASS: required metrics present: http_server_request_duration_seconds_bucket, messenger_jobs_processed_total, messenger_job_duration_seconds_bucket, messenger_attachment_time_to_ready_seconds_bucket, messenger_queue_depth, messenger_queue_oldest_age_seconds, messenger_worker_busy, messenger_worker_capacity, db_client_connection_count, db_client_connection_pending_requests, process_cpu_time_seconds_total, process_memory_usage_bytes, otelcol_receiver_accepted_spans_total, otelcol_exporter_sent_spans_total, otelcol_exporter_queue_size +PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy) diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..b2e8607 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,36 @@ +# 2026-11-11T08:09:52Z scripts/restart.sh +$ docker compose ps (before) +mtusi-lab7-api-1-1 Up 51 minutes (healthy) +mtusi-lab7-api-2-1 Up 51 minutes (healthy) +mtusi-lab7-grafana-1 Up About an hour (healthy) +mtusi-lab7-loki-1 Up About an hour (healthy) +mtusi-lab7-minio-1 Up About an hour (healthy) +mtusi-lab7-nginx-1 Up 43 minutes (healthy) +mtusi-lab7-openbao-1 Up About a minute (healthy) +mtusi-lab7-otel-collector-1 Up 42 minutes +mtusi-lab7-postgres-1 Up About an hour (healthy) +mtusi-lab7-prometheus-1 Up About an hour (healthy) +mtusi-lab7-rabbitmq-1 Up 40 minutes (healthy) +mtusi-lab7-tempo-1 Up About an hour (healthy) +mtusi-lab7-valkey-1 Up About an hour (healthy) +mtusi-lab7-worker-1 Up 50 minutes (healthy) +mtusi-lab7-worker-2 Up 51 seconds (healthy) +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +$ docker compose ps (after) +mtusi-lab7-api-1-1 Up 2 seconds (health: starting) +mtusi-lab7-api-2-1 Up 1 second (health: starting) +mtusi-lab7-grafana-1 Up 4 seconds (health: starting) +mtusi-lab7-loki-1 Up 2 seconds (health: starting) +mtusi-lab7-minio-1 Up 7 seconds (healthy) +mtusi-lab7-nginx-1 Up 2 seconds (health: starting) +mtusi-lab7-openbao-1 Up 7 seconds (healthy) +mtusi-lab7-otel-collector-1 Up 4 seconds +mtusi-lab7-postgres-1 Up 7 seconds (healthy) +mtusi-lab7-prometheus-1 Up 4 seconds (health: starting) +mtusi-lab7-rabbitmq-1 Up 6 seconds (healthy) +mtusi-lab7-tempo-1 Up 4 seconds (health: starting) +mtusi-lab7-valkey-1 Up 7 seconds (healthy) +mtusi-lab7-worker-1 Up 1 second (health: starting) +mtusi-lab7-worker-2 Up 1 second (health: starting) diff --git a/evidence/secret_rotation.txt b/evidence/secret_rotation.txt new file mode 100644 index 0000000..757f5c5 --- /dev/null +++ b/evidence/secret_rotation.txt @@ -0,0 +1,8 @@ +11:08:20 before: old password accepted, 2 app_worker connections open +11:08:21 rotated: app_worker password changed in PostgreSQL and OpenBao (messenger/worker, new KV version) +11:08:30 both workers loaded the new KV version and switched the pool (10 s after rotation) +11:08:30 terminated all existing app_worker sessions: the pools must reconnect with the new password +11:08:36 3 uploads after the forced reconnect are ready; 2 app_worker connections open again +11:08:37 probe: old password -> rejected: FATAL: password authentication failed for user "app_worker" | new password -> accepted +11:08:37 traffic: 116 requests, unexpected errors 0, uploads 20; failed jobs since rotation: 0 +PASS: app_worker password rotated via OpenBao without rebuild/restart; old password revoked diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..c9a6fad --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt" +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/smoke_check_validate_security.txt b/evidence/smoke_check_validate_security.txt new file mode 100644 index 0000000..a35bcdc --- /dev/null +++ b/evidence/smoke_check_validate_security.txt @@ -0,0 +1,9 @@ +# 2026-11-11T08:08:18Z (OpenBao sealed at this moment: running services use the secrets they already hold) +$ make check +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +$ make validate +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +$ make test CA_FILE=/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +$ make test-security CA_FILE=/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. diff --git a/evidence/tls_audit.json b/evidence/tls_audit.json new file mode 100644 index 0000000..d4c009d --- /dev/null +++ b/evidence/tls_audit.json @@ -0,0 +1,1082 @@ +[ + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "foreign CA (claims lb identity)", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "same CA, identity intruder", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "same CA, identity worker", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "same CA, identity api", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "lb.client.messenger", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-1:8080", + "case": "plaintext client", + "expect": "reject", + "observed": "server answered b''", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "foreign CA (claims lb identity)", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "same CA, identity intruder", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "same CA, identity worker", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "same CA, identity api", + "expect": "reject", + "observed": "rejected: closed without response", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "lb.client.messenger", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "A", + "link": "LB -> api-2:8080", + "case": "plaintext client", + "expect": "reject", + "observed": "server answered b''", + "pass": true + }, + { + "section": "I", + "link": "client -> nginx:8443", + "case": "trusted CA, hostname nginx", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "I", + "link": "client -> nginx:8443", + "case": "untrusted CA in client trust store", + "expect": "reject", + "observed": "rejected: server cert verify failed: unable to get local issuer certificate", + "pass": true + }, + { + "section": "I", + "link": "client -> nginx:8443", + "case": "wrong hostname (evil.example)", + "expect": "reject", + "observed": "rejected: server cert verify failed: Hostname mismatch, certificate is not valid for 'evil.example'.", + "pass": true + }, + { + "section": "I", + "link": "client -> nginx:8443", + "case": "plaintext client", + "expect": "reject", + "observed": "server answered b'HTTP/1.1 400 Bad Request'", + "pass": true + }, + { + "section": "B", + "link": "svc -> postgres:5432", + "case": "plaintext (sslmode=disable)", + "expect": "reject", + "observed": "OperationalError: connection failed: connection to server at \"172.22.0.9\", port 5432 failed: FATAL: pg_hba.conf rejects connection for host \"172.22.0.17\", user", + "pass": true + }, + { + "section": "B", + "link": "svc -> postgres:5432", + "case": "foreign CA as root (verify-full)", + "expect": "reject", + "observed": "OperationalError: connection failed: connection to server at \"172.22.0.9\", port 5432 failed: SSL error: certificate verify failed", + "pass": true + }, + { + "section": "B", + "link": "svc -> postgres:5432", + "case": "hostname mismatch (IP, verify-full)", + "expect": "reject", + "observed": "connection failed: connection to server at \"172.22.0.9\", port 5432 failed: server certificate for \"postgres\" (and 1 other name) does not match host name \"172.22", + "pass": true + }, + { + "section": "B", + "link": "svc -> postgres:5432", + "case": "TLS verify-full + app_api password", + "expect": "accept", + "observed": "connected", + "pass": true + }, + { + "section": "B", + "link": "svc -> postgres:5432", + "case": "wrong password", + "expect": "reject", + "observed": "connection failed: connection to server at \"172.22.0.9\", port 5432 failed: FATAL: password authentication failed for user \"app_api\"", + "pass": true + }, + { + "section": "B", + "link": "app_worker privileges", + "case": "SELECT count(*) FROM users", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "B", + "link": "app_worker privileges", + "case": "SELECT count(*) FROM messages", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "B", + "link": "app_worker privileges", + "case": "SELECT count(*) FROM sessions", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "B", + "link": "app_worker privileges", + "case": "DELETE FROM attachments WHERE false", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "B", + "link": "app_worker privileges", + "case": "SELECT count(*) FROM attachment_jobs", + "expect": "allow", + "observed": "executed", + "pass": true + }, + { + "section": "B", + "link": "app_api privileges", + "case": "SELECT count(*) FROM users", + "expect": "allow", + "observed": "executed", + "pass": true + }, + { + "section": "B", + "link": "app_api privileges", + "case": "CREATE TABLE audit_probe (x int)", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "B", + "link": "app_api privileges", + "case": "ALTER ROLE app_worker PASSWORD 'x'", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "B", + "link": "app_api privileges", + "case": "SELECT count(*) FROM schema_migrations", + "expect": "deny", + "observed": "InsufficientPrivilege", + "pass": true + }, + { + "section": "C", + "link": "svc -> minio:9000", + "case": "foreign CA as trust anchor", + "expect": "reject", + "observed": "SSLError", + "pass": true + }, + { + "section": "C", + "link": "svc -> minio:9000", + "case": "plaintext http://", + "expect": "reject", + "observed": "ConnectionClosedError", + "pass": true + }, + { + "section": "C", + "link": "worker S3 rights", + "case": "PUT attachments/*", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "C", + "link": "worker S3 rights", + "case": "GET attachments/*", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "worker S3 rights", + "case": "DELETE attachments/*", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "worker S3 rights", + "case": "PUT staging/*", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "worker S3 rights", + "case": "LIST bucket", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "worker S3 rights", + "case": "PUT outside prefixes", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "api S3 rights", + "case": "PUT staging/*", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "C", + "link": "api S3 rights", + "case": "PUT attachments/* (only worker publishes)", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "api S3 rights", + "case": "GET attachments/*", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "C", + "link": "api S3 rights", + "case": "LIST bucket (reconciler)", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "C", + "link": "api S3 rights", + "case": "create bucket", + "expect": "deny", + "observed": "AccessDenied", + "pass": true + }, + { + "section": "C", + "link": "api S3 rights", + "case": "cleanup audit objects", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "D", + "link": "svc -> rabbitmq:5672", + "case": "plaintext client", + "expect": "reject", + "observed": "connection failed: ConnectionRefusedError", + "pass": true + }, + { + "section": "D", + "link": "svc -> rabbitmq:5671", + "case": "no client cert", + "expect": "reject", + "observed": "AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV13_ALERT_", + "pass": true + }, + { + "section": "D", + "link": "svc -> rabbitmq:5671", + "case": "foreign CA (claims worker name)", + "expect": "reject", + "observed": "AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV1_ALERT_U", + "pass": true + }, + { + "section": "D", + "link": "svc -> rabbitmq:5671", + "case": "same CA, unknown identity intruder", + "expect": "reject", + "observed": "ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech", + "pass": true + }, + { + "section": "D", + "link": "svc -> rabbitmq:5671", + "case": "same CA, identity lb", + "expect": "reject", + "observed": "ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech", + "pass": true + }, + { + "section": "D", + "link": "api AMQP rights", + "case": "publish to exchange attachments", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "D", + "link": "api AMQP rights", + "case": "consume attachments.process", + "expect": "deny", + "observed": "ChannelAccessRefused ACCESS_REFUSED - read access to queue 'attachments.process' in vhost '", + "pass": true + }, + { + "section": "D", + "link": "api AMQP rights", + "case": "declare a queue (configure)", + "expect": "deny", + "observed": "ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes", + "pass": true + }, + { + "section": "D", + "link": "api AMQP rights", + "case": "publish via default exchange", + "expect": "deny", + "observed": "ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'amq.default' in vhost 'mess", + "pass": true + }, + { + "section": "D", + "link": "worker AMQP rights", + "case": "consume attachments.process", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "D", + "link": "worker AMQP rights", + "case": "publish to exchange attachments", + "expect": "deny", + "observed": "ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'attachments' in vhost 'mess", + "pass": true + }, + { + "section": "D", + "link": "worker AMQP rights", + "case": "declare a queue (configure)", + "expect": "deny", + "observed": "ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "foreign CA", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "root CA identity lb (not ingest CA)", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "root CA identity api (not ingest CA)", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "telemetry CA (not ingest CA)", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "ingest CA api.otel", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "E", + "link": "svc -> otel-collector:4318", + "case": "ingest CA worker.otel", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "foreign CA", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "root CA identity api", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "ingest CA (apps cannot bypass collector)", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "telemetry CA collector.out", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "telemetry CA grafana.out", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "F", + "link": "-> prometheus:9090", + "case": "telemetry CA operator", + "expect": "accept", + "observed": "accepted: HTTP/1.1 200 OK", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "foreign CA", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "root CA identity api", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "ingest CA (apps cannot bypass collector)", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "telemetry CA collector.out", + "expect": "accept", + "observed": "accepted: HTTP/1.1 503 Service Unavailable", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "telemetry CA grafana.out", + "expect": "accept", + "observed": "accepted: HTTP/1.1 503 Service Unavailable", + "pass": true + }, + { + "section": "F", + "link": "-> loki:3100", + "case": "telemetry CA operator", + "expect": "accept", + "observed": "accepted: HTTP/1.1 503 Service Unavailable", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "foreign CA", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "root CA identity api", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "ingest CA (apps cannot bypass collector)", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "telemetry CA collector.out", + "expect": "accept", + "observed": "accepted: HTTP/1.1 503 Service Unavailable", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "telemetry CA grafana.out", + "expect": "accept", + "observed": "accepted: HTTP/1.1 503 Service Unavailable", + "pass": true + }, + { + "section": "F", + "link": "-> tempo:3200", + "case": "telemetry CA operator", + "expect": "accept", + "observed": "accepted: HTTP/1.1 503 Service Unavailable", + "pass": true + }, + { + "section": "F", + "link": "collector -> tempo:4317 (gRPC)", + "case": "no client cert", + "expect": "reject", + "observed": "rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED", + "pass": true + }, + { + "section": "F", + "link": "collector -> tempo:4317 (gRPC)", + "case": "ingest CA", + "expect": "reject", + "observed": "rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA", + "pass": true + }, + { + "section": "F", + "link": "collector -> tempo:4317 (gRPC)", + "case": "telemetry CA collector.out", + "expect": "accept", + "observed": "accepted: \u0000\u0000\u0006\u0004\u0000\u0000\u0000\u0000\u0000\u0000\u0005\u0000\u0000@\u0000", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "plaintext client", + "expect": "reject", + "observed": "connection failed: ConnectionResetError", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "foreign CA as trust anchor", + "expect": "deny", + "observed": "ConnectionError Error 1 connecting to valkey:6379. [SSL: CERTIFICATE_VERIFY_", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "no authentication (default user off)", + "expect": "deny", + "observed": "AuthenticationError Authentication required.", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "user api, wrong password", + "expect": "deny", + "observed": "AuthenticationError invalid username-password pair or user is disabled.", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "api: SET/GET session:*", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "api: INCR ratelimit:*", + "expect": "allow", + "observed": "ok", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "api: GET outside allowed keys", + "expect": "deny", + "observed": "NoPermissionError No permissions to access a key", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "api: FLUSHALL", + "expect": "deny", + "observed": "NoPermissionError User api has no permissions to run the 'flushall' command", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "api: CONFIG GET", + "expect": "deny", + "observed": "NoPermissionError User api has no permissions to run the 'config|get' command", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "api: KEYS *", + "expect": "deny", + "observed": "NoPermissionError User api has no permissions to run the 'keys' command", + "pass": true + }, + { + "section": "G", + "link": "api -> valkey:6379", + "case": "health: GET session:*", + "expect": "deny", + "observed": "NoPermissionError User health has no permissions to run the 'get' command", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: no client cert", + "expect": "reject", + "observed": "HTTP 400 token=False", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: same CA, identity intruder", + "expect": "reject", + "observed": "HTTP 400 token=False", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: worker cert, role api", + "expect": "reject", + "observed": "HTTP 400 token=False", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: api cert, role worker", + "expect": "reject", + "observed": "HTTP 400 token=False", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: lb cert, role api", + "expect": "reject", + "observed": "HTTP 400 token=False", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: foreign CA (claims worker name)", + "expect": "reject", + "observed": "HTTP 400 token=False", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: worker cert, role worker", + "expect": "accept", + "observed": "HTTP 200 token=True", + "pass": true + }, + { + "section": "H", + "link": "worker token", + "case": "policies / ttl", + "expect": "scoped", + "observed": "policies=['default', 'messenger-worker'] ttl=900s", + "pass": true + }, + { + "section": "H", + "link": "worker token", + "case": "read own messenger/worker", + "expect": "allow", + "observed": "HTTP 200", + "pass": true + }, + { + "section": "H", + "link": "worker token", + "case": "read messenger/api", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "worker token", + "case": "write own secret", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "worker token", + "case": "write own policy", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "worker token", + "case": "list auth methods (admin)", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "svc -> openbao:8200", + "case": "cert login: api cert, role api", + "expect": "accept", + "observed": "HTTP 200 token=True", + "pass": true + }, + { + "section": "H", + "link": "api token", + "case": "policies / ttl", + "expect": "scoped", + "observed": "policies=['default', 'messenger-api'] ttl=900s", + "pass": true + }, + { + "section": "H", + "link": "api token", + "case": "read own messenger/api", + "expect": "allow", + "observed": "HTTP 200", + "pass": true + }, + { + "section": "H", + "link": "api token", + "case": "read messenger/worker", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "api token", + "case": "write own secret", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "api token", + "case": "write own policy", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "H", + "link": "api token", + "case": "list auth methods (admin)", + "expect": "deny", + "observed": "HTTP 403", + "pass": true + }, + { + "section": "J", + "link": "operator -> grafana:3000", + "case": "HTTPS without login", + "expect": "reject", + "observed": "accepted: HTTP/1.1 401 Unauthorized", + "pass": true + }, + { + "section": "J", + "link": "operator -> grafana:3000", + "case": "plaintext client", + "expect": "reject", + "observed": "server answered b'HTTP/1.0 400 Bad Request'", + "pass": true + }, + { + "section": "K", + "link": "rabbitmq:25672", + "case": "Erlang distribution", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "rabbitmq:4369", + "case": "epmd", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "rabbitmq:15672", + "case": "management UI", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "loki:9096", + "case": "Loki internal gRPC", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "tempo:9095", + "case": "Tempo internal gRPC", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "otel-collector:8888", + "case": "collector self-metrics", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "otel-collector:13133", + "case": "collector health", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "otel-collector:4317", + "case": "collector OTLP/gRPC (disabled)", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "api-1:8081", + "case": "API loopback health", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "api-2:8081", + "case": "API loopback health", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "openbao:8201", + "case": "OpenBao cluster port", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "minio:9001", + "case": "MinIO console", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + }, + { + "section": "K", + "link": "postgres:5433", + "case": "no second PostgreSQL listener", + "expect": "closed", + "observed": "ConnectionRefusedError", + "pass": true + } +] \ No newline at end of file diff --git a/evidence/tls_audit.txt b/evidence/tls_audit.txt new file mode 100644 index 0000000..e1c0277 --- /dev/null +++ b/evidence/tls_audit.txt @@ -0,0 +1,124 @@ +PASS A LB -> api-1:8080 no client cert expect=reject rejected: closed without response +PASS A LB -> api-1:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response +PASS A LB -> api-1:8080 same CA, identity intruder expect=reject rejected: closed without response +PASS A LB -> api-1:8080 same CA, identity worker expect=reject rejected: closed without response +PASS A LB -> api-1:8080 same CA, identity api expect=reject rejected: closed without response +PASS A LB -> api-1:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK +PASS A LB -> api-1:8080 plaintext client expect=reject server answered b'' +PASS A LB -> api-2:8080 no client cert expect=reject rejected: closed without response +PASS A LB -> api-2:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response +PASS A LB -> api-2:8080 same CA, identity intruder expect=reject rejected: closed without response +PASS A LB -> api-2:8080 same CA, identity worker expect=reject rejected: closed without response +PASS A LB -> api-2:8080 same CA, identity api expect=reject rejected: closed without response +PASS A LB -> api-2:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK +PASS A LB -> api-2:8080 plaintext client expect=reject server answered b'' +PASS I client -> nginx:8443 trusted CA, hostname nginx expect=accept accepted: HTTP/1.1 200 OK +PASS I client -> nginx:8443 untrusted CA in client trust store expect=reject rejected: server cert verify failed: unable to get local issuer certificate +PASS I client -> nginx:8443 wrong hostname (evil.example) expect=reject rejected: server cert verify failed: Hostname mismatch, certificate is not valid for 'evil.example'. +PASS I client -> nginx:8443 plaintext client expect=reject server answered b'HTTP/1.1 400 Bad Request' +PASS B svc -> postgres:5432 plaintext (sslmode=disable) expect=reject OperationalError: connection failed: connection to server at "172.22.0.10", port 5432 failed: FATAL: pg_hba.c +PASS B svc -> postgres:5432 foreign CA as root (verify-full) expect=reject OperationalError: connection failed: connection to server at "172.22.0.10", port 5432 failed: SSL error: certi +PASS B svc -> postgres:5432 hostname mismatch (IP, verify-full) expect=reject connection failed: connection to server at "172.22.0.10", port 5432 failed: server certificate for "postgres" +PASS B svc -> postgres:5432 TLS verify-full + app_api password expect=accept connected +PASS B svc -> postgres:5432 wrong password expect=reject connection failed: connection to server at "172.22.0.10", port 5432 failed: FATAL: password authentication fa +PASS B app_worker privileges SELECT count(*) FROM users expect=deny InsufficientPrivilege +PASS B app_worker privileges SELECT count(*) FROM messages expect=deny InsufficientPrivilege +PASS B app_worker privileges SELECT count(*) FROM sessions expect=deny InsufficientPrivilege +PASS B app_worker privileges DELETE FROM attachments WHERE false expect=deny InsufficientPrivilege +PASS B app_worker privileges SELECT count(*) FROM attachment_jobs expect=allow executed +PASS B app_api privileges SELECT count(*) FROM users expect=allow executed +PASS B app_api privileges CREATE TABLE audit_probe (x int) expect=deny InsufficientPrivilege +PASS B app_api privileges ALTER ROLE app_worker PASSWORD 'x' expect=deny InsufficientPrivilege +PASS B app_api privileges SELECT count(*) FROM schema_migrations expect=deny InsufficientPrivilege +PASS C svc -> minio:9000 foreign CA as trust anchor expect=reject SSLError +PASS C svc -> minio:9000 plaintext http:// expect=reject ConnectionClosedError +PASS C worker S3 rights PUT attachments/* expect=allow ok +PASS C worker S3 rights GET attachments/* expect=deny AccessDenied +PASS C worker S3 rights DELETE attachments/* expect=deny AccessDenied +PASS C worker S3 rights PUT staging/* expect=deny AccessDenied +PASS C worker S3 rights LIST bucket expect=deny AccessDenied +PASS C worker S3 rights PUT outside prefixes expect=deny AccessDenied +PASS C api S3 rights PUT staging/* expect=allow ok +PASS C api S3 rights PUT attachments/* (only worker publishes) expect=deny AccessDenied +PASS C api S3 rights GET attachments/* expect=allow ok +PASS C api S3 rights LIST bucket (reconciler) expect=allow ok +PASS C api S3 rights create bucket expect=deny AccessDenied +PASS C api S3 rights cleanup audit objects expect=allow ok +PASS D svc -> rabbitmq:5672 plaintext client expect=reject connection failed: ConnectionRefusedError +PASS D svc -> rabbitmq:5671 no client cert expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV13_ALERT_ +PASS D svc -> rabbitmq:5671 foreign CA (claims worker name) expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV1_ALERT_U +PASS D svc -> rabbitmq:5671 same CA, unknown identity intruder expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech +PASS D svc -> rabbitmq:5671 same CA, identity lb expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech +PASS D api AMQP rights publish to exchange attachments expect=allow ok +PASS D api AMQP rights consume attachments.process expect=deny ChannelAccessRefused ACCESS_REFUSED - read access to queue 'attachments.process' in vhost ' +PASS D api AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes +PASS D api AMQP rights publish via default exchange expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'amq.default' in vhost 'mess +PASS D worker AMQP rights consume attachments.process expect=allow ok +PASS D worker AMQP rights publish to exchange attachments expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'attachments' in vhost 'mess +PASS D worker AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes +PASS E svc -> otel-collector:4318 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS E svc -> otel-collector:4318 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 root CA identity lb (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 root CA identity api (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 telemetry CA (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 ingest CA api.otel expect=accept accepted: HTTP/1.1 200 OK +PASS E svc -> otel-collector:4318 ingest CA worker.otel expect=accept accepted: HTTP/1.1 200 OK +PASS F -> prometheus:9090 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F -> prometheus:9090 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> prometheus:9090 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> prometheus:9090 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> prometheus:9090 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> prometheus:9090 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> prometheus:9090 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK +PASS F -> loki:3100 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F -> loki:3100 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> loki:3100 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> loki:3100 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> loki:3100 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> loki:3100 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> loki:3100 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK +PASS F -> tempo:3200 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F -> tempo:3200 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> tempo:3200 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> tempo:3200 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> tempo:3200 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> tempo:3200 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> tempo:3200 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK +PASS F collector -> tempo:4317 (gRPC) no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F collector -> tempo:4317 (gRPC) ingest CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F collector -> tempo:4317 (gRPC) telemetry CA collector.out expect=accept accepted: @ +PASS G api -> valkey:6379 plaintext client expect=reject connection failed: ConnectionResetError +PASS G api -> valkey:6379 foreign CA as trust anchor expect=deny ConnectionError Error 1 connecting to valkey:6379. [SSL: CERTIFICATE_VERIFY_ +PASS G api -> valkey:6379 no authentication (default user off) expect=deny AuthenticationError Authentication required. +PASS G api -> valkey:6379 user api, wrong password expect=deny AuthenticationError invalid username-password pair or user is disabled. +PASS G api -> valkey:6379 api: SET/GET session:* expect=allow ok +PASS G api -> valkey:6379 api: INCR ratelimit:* expect=allow ok +PASS G api -> valkey:6379 api: GET outside allowed keys expect=deny NoPermissionError No permissions to access a key +PASS G api -> valkey:6379 api: FLUSHALL expect=deny NoPermissionError User api has no permissions to run the 'flushall' command +PASS G api -> valkey:6379 api: CONFIG GET expect=deny NoPermissionError User api has no permissions to run the 'config|get' command +PASS G api -> valkey:6379 api: KEYS * expect=deny NoPermissionError User api has no permissions to run the 'keys' command +PASS G api -> valkey:6379 health: GET session:* expect=deny NoPermissionError User health has no permissions to run the 'get' command +PASS H svc -> openbao:8200 cert login: no client cert expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: same CA, identity intruder expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: worker cert, role api expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: api cert, role worker expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: lb cert, role api expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: foreign CA (claims worker name) expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: worker cert, role worker expect=accept HTTP 200 token=True +PASS H worker token policies / ttl expect=scoped policies=['default', 'messenger-worker'] ttl=900s +PASS H worker token read own messenger/worker expect=allow HTTP 200 +PASS H worker token read messenger/api expect=deny HTTP 403 +PASS H worker token write own secret expect=deny HTTP 403 +PASS H worker token write own policy expect=deny HTTP 403 +PASS H worker token list auth methods (admin) expect=deny HTTP 403 +PASS H svc -> openbao:8200 cert login: api cert, role api expect=accept HTTP 200 token=True +PASS H api token policies / ttl expect=scoped policies=['default', 'messenger-api'] ttl=900s +PASS H api token read own messenger/api expect=allow HTTP 200 +PASS H api token read messenger/worker expect=deny HTTP 403 +PASS H api token write own secret expect=deny HTTP 403 +PASS H api token write own policy expect=deny HTTP 403 +PASS H api token list auth methods (admin) expect=deny HTTP 403 +PASS J operator -> grafana:3000 HTTPS without login expect=reject accepted: HTTP/1.1 401 Unauthorized +PASS J operator -> grafana:3000 plaintext client expect=reject server answered b'HTTP/1.0 400 Bad Request' + +122 checks, 0 failed diff --git a/evidence/tls_audit_final.txt b/evidence/tls_audit_final.txt new file mode 100644 index 0000000..061b8f3 --- /dev/null +++ b/evidence/tls_audit_final.txt @@ -0,0 +1,139 @@ + Container mtusi-lab7-worker-run-dcda01946f3d Creating + Container mtusi-lab7-worker-run-dcda01946f3d Created +PASS A LB -> api-1:8080 no client cert expect=reject rejected: closed without response +PASS A LB -> api-1:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response +PASS A LB -> api-1:8080 same CA, identity intruder expect=reject rejected: closed without response +PASS A LB -> api-1:8080 same CA, identity worker expect=reject rejected: closed without response +PASS A LB -> api-1:8080 same CA, identity api expect=reject rejected: closed without response +PASS A LB -> api-1:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK +PASS A LB -> api-1:8080 plaintext client expect=reject server answered b'' +PASS A LB -> api-2:8080 no client cert expect=reject rejected: closed without response +PASS A LB -> api-2:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response +PASS A LB -> api-2:8080 same CA, identity intruder expect=reject rejected: closed without response +PASS A LB -> api-2:8080 same CA, identity worker expect=reject rejected: closed without response +PASS A LB -> api-2:8080 same CA, identity api expect=reject rejected: closed without response +PASS A LB -> api-2:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK +PASS A LB -> api-2:8080 plaintext client expect=reject server answered b'' +PASS I client -> nginx:8443 trusted CA, hostname nginx expect=accept accepted: HTTP/1.1 200 OK +PASS I client -> nginx:8443 untrusted CA in client trust store expect=reject rejected: server cert verify failed: unable to get local issuer certificate +PASS I client -> nginx:8443 wrong hostname (evil.example) expect=reject rejected: server cert verify failed: Hostname mismatch, certificate is not valid for 'evil.example'. +PASS I client -> nginx:8443 plaintext client expect=reject server answered b'HTTP/1.1 400 Bad Request' +PASS B svc -> postgres:5432 plaintext (sslmode=disable) expect=reject OperationalError: connection failed: connection to server at "172.22.0.9", port 5432 failed: FATAL: pg_hba.co +PASS B svc -> postgres:5432 foreign CA as root (verify-full) expect=reject OperationalError: connection failed: connection to server at "172.22.0.9", port 5432 failed: SSL error: certif +PASS B svc -> postgres:5432 hostname mismatch (IP, verify-full) expect=reject connection failed: connection to server at "172.22.0.9", port 5432 failed: server certificate for "postgres" ( +PASS B svc -> postgres:5432 TLS verify-full + app_api password expect=accept connected +PASS B svc -> postgres:5432 wrong password expect=reject connection failed: connection to server at "172.22.0.9", port 5432 failed: FATAL: password authentication fai +PASS B app_worker privileges SELECT count(*) FROM users expect=deny InsufficientPrivilege +PASS B app_worker privileges SELECT count(*) FROM messages expect=deny InsufficientPrivilege +PASS B app_worker privileges SELECT count(*) FROM sessions expect=deny InsufficientPrivilege +PASS B app_worker privileges DELETE FROM attachments WHERE false expect=deny InsufficientPrivilege +PASS B app_worker privileges SELECT count(*) FROM attachment_jobs expect=allow executed +PASS B app_api privileges SELECT count(*) FROM users expect=allow executed +PASS B app_api privileges CREATE TABLE audit_probe (x int) expect=deny InsufficientPrivilege +PASS B app_api privileges ALTER ROLE app_worker PASSWORD 'x' expect=deny InsufficientPrivilege +PASS B app_api privileges SELECT count(*) FROM schema_migrations expect=deny InsufficientPrivilege +PASS C svc -> minio:9000 foreign CA as trust anchor expect=reject SSLError +PASS C svc -> minio:9000 plaintext http:// expect=reject ConnectionClosedError +PASS C worker S3 rights PUT attachments/* expect=allow ok +PASS C worker S3 rights GET attachments/* expect=deny AccessDenied +PASS C worker S3 rights DELETE attachments/* expect=deny AccessDenied +PASS C worker S3 rights PUT staging/* expect=deny AccessDenied +PASS C worker S3 rights LIST bucket expect=deny AccessDenied +PASS C worker S3 rights PUT outside prefixes expect=deny AccessDenied +PASS C api S3 rights PUT staging/* expect=allow ok +PASS C api S3 rights PUT attachments/* (only worker publishes) expect=deny AccessDenied +PASS C api S3 rights GET attachments/* expect=allow ok +PASS C api S3 rights LIST bucket (reconciler) expect=allow ok +PASS C api S3 rights create bucket expect=deny AccessDenied +PASS C api S3 rights cleanup audit objects expect=allow ok +PASS D svc -> rabbitmq:5672 plaintext client expect=reject connection failed: ConnectionRefusedError +PASS D svc -> rabbitmq:5671 no client cert expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV13_ALERT_ +PASS D svc -> rabbitmq:5671 foreign CA (claims worker name) expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV1_ALERT_U +PASS D svc -> rabbitmq:5671 same CA, unknown identity intruder expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech +PASS D svc -> rabbitmq:5671 same CA, identity lb expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech +PASS D api AMQP rights publish to exchange attachments expect=allow ok +PASS D api AMQP rights consume attachments.process expect=deny ChannelAccessRefused ACCESS_REFUSED - read access to queue 'attachments.process' in vhost ' +PASS D api AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes +PASS D api AMQP rights publish via default exchange expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'amq.default' in vhost 'mess +PASS D worker AMQP rights consume attachments.process expect=allow ok +PASS D worker AMQP rights publish to exchange attachments expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'attachments' in vhost 'mess +PASS D worker AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes +PASS E svc -> otel-collector:4318 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS E svc -> otel-collector:4318 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 root CA identity lb (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 root CA identity api (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 telemetry CA (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS E svc -> otel-collector:4318 ingest CA api.otel expect=accept accepted: HTTP/1.1 200 OK +PASS E svc -> otel-collector:4318 ingest CA worker.otel expect=accept accepted: HTTP/1.1 200 OK +PASS F -> prometheus:9090 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F -> prometheus:9090 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> prometheus:9090 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> prometheus:9090 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> prometheus:9090 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> prometheus:9090 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK +PASS F -> prometheus:9090 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK +PASS F -> loki:3100 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F -> loki:3100 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> loki:3100 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> loki:3100 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> loki:3100 telemetry CA collector.out expect=accept accepted: HTTP/1.1 503 Service Unavailable +PASS F -> loki:3100 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 503 Service Unavailable +PASS F -> loki:3100 telemetry CA operator expect=accept accepted: HTTP/1.1 503 Service Unavailable +PASS F -> tempo:3200 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F -> tempo:3200 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> tempo:3200 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> tempo:3200 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F -> tempo:3200 telemetry CA collector.out expect=accept accepted: HTTP/1.1 503 Service Unavailable +PASS F -> tempo:3200 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 503 Service Unavailable +PASS F -> tempo:3200 telemetry CA operator expect=accept accepted: HTTP/1.1 503 Service Unavailable +PASS F collector -> tempo:4317 (gRPC) no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED +PASS F collector -> tempo:4317 (gRPC) ingest CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA +PASS F collector -> tempo:4317 (gRPC) telemetry CA collector.out expect=accept accepted: @ +PASS G api -> valkey:6379 plaintext client expect=reject connection failed: ConnectionResetError +PASS G api -> valkey:6379 foreign CA as trust anchor expect=deny ConnectionError Error 1 connecting to valkey:6379. [SSL: CERTIFICATE_VERIFY_ +PASS G api -> valkey:6379 no authentication (default user off) expect=deny AuthenticationError Authentication required. +PASS G api -> valkey:6379 user api, wrong password expect=deny AuthenticationError invalid username-password pair or user is disabled. +PASS G api -> valkey:6379 api: SET/GET session:* expect=allow ok +PASS G api -> valkey:6379 api: INCR ratelimit:* expect=allow ok +PASS G api -> valkey:6379 api: GET outside allowed keys expect=deny NoPermissionError No permissions to access a key +PASS G api -> valkey:6379 api: FLUSHALL expect=deny NoPermissionError User api has no permissions to run the 'flushall' command +PASS G api -> valkey:6379 api: CONFIG GET expect=deny NoPermissionError User api has no permissions to run the 'config|get' command +PASS G api -> valkey:6379 api: KEYS * expect=deny NoPermissionError User api has no permissions to run the 'keys' command +PASS G api -> valkey:6379 health: GET session:* expect=deny NoPermissionError User health has no permissions to run the 'get' command +PASS H svc -> openbao:8200 cert login: no client cert expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: same CA, identity intruder expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: worker cert, role api expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: api cert, role worker expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: lb cert, role api expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: foreign CA (claims worker name) expect=reject HTTP 400 token=False +PASS H svc -> openbao:8200 cert login: worker cert, role worker expect=accept HTTP 200 token=True +PASS H worker token policies / ttl expect=scoped policies=['default', 'messenger-worker'] ttl=900s +PASS H worker token read own messenger/worker expect=allow HTTP 200 +PASS H worker token read messenger/api expect=deny HTTP 403 +PASS H worker token write own secret expect=deny HTTP 403 +PASS H worker token write own policy expect=deny HTTP 403 +PASS H worker token list auth methods (admin) expect=deny HTTP 403 +PASS H svc -> openbao:8200 cert login: api cert, role api expect=accept HTTP 200 token=True +PASS H api token policies / ttl expect=scoped policies=['default', 'messenger-api'] ttl=900s +PASS H api token read own messenger/api expect=allow HTTP 200 +PASS H api token read messenger/worker expect=deny HTTP 403 +PASS H api token write own secret expect=deny HTTP 403 +PASS H api token write own policy expect=deny HTTP 403 +PASS H api token list auth methods (admin) expect=deny HTTP 403 +PASS J operator -> grafana:3000 HTTPS without login expect=reject accepted: HTTP/1.1 401 Unauthorized +PASS J operator -> grafana:3000 plaintext client expect=reject server answered b'HTTP/1.0 400 Bad Request' +PASS K rabbitmq:25672 Erlang distribution expect=closed ConnectionRefusedError +PASS K rabbitmq:4369 epmd expect=closed ConnectionRefusedError +PASS K rabbitmq:15672 management UI expect=closed ConnectionRefusedError +PASS K loki:9096 Loki internal gRPC expect=closed ConnectionRefusedError +PASS K tempo:9095 Tempo internal gRPC expect=closed ConnectionRefusedError +PASS K otel-collector:8888 collector self-metrics expect=closed ConnectionRefusedError +PASS K otel-collector:13133 collector health expect=closed ConnectionRefusedError +PASS K otel-collector:4317 collector OTLP/gRPC (disabled) expect=closed ConnectionRefusedError +PASS K api-1:8081 API loopback health expect=closed ConnectionRefusedError +PASS K api-2:8081 API loopback health expect=closed ConnectionRefusedError +PASS K openbao:8201 OpenBao cluster port expect=closed ConnectionRefusedError +PASS K minio:9001 MinIO console expect=closed ConnectionRefusedError +PASS K postgres:5433 no second PostgreSQL listener expect=closed ConnectionRefusedError + +135 checks, 0 failed diff --git a/evidence/tls_audit_side_channels.txt b/evidence/tls_audit_side_channels.txt new file mode 100644 index 0000000..0e3abaa --- /dev/null +++ b/evidence/tls_audit_side_channels.txt @@ -0,0 +1,15 @@ +PASS K rabbitmq:25672 Erlang distribution expect=closed ConnectionRefusedError +PASS K rabbitmq:4369 epmd expect=closed ConnectionRefusedError +PASS K rabbitmq:15672 management UI expect=closed ConnectionRefusedError +PASS K loki:9096 Loki internal gRPC expect=closed ConnectionRefusedError +PASS K tempo:9095 Tempo internal gRPC expect=closed ConnectionRefusedError +PASS K otel-collector:8888 collector self-metrics expect=closed ConnectionRefusedError +PASS K otel-collector:13133 collector health expect=closed ConnectionRefusedError +PASS K otel-collector:4317 collector OTLP/gRPC (disabled) expect=closed ConnectionRefusedError +PASS K api-1:8081 API loopback health expect=closed ConnectionRefusedError +PASS K api-2:8081 API loopback health expect=closed ConnectionRefusedError +PASS K openbao:8201 OpenBao cluster port expect=closed ConnectionRefusedError +PASS K minio:9001 MinIO console expect=closed ConnectionRefusedError +PASS K postgres:5433 no second PostgreSQL listener expect=closed ConnectionRefusedError + +13 checks, 0 failed diff --git a/evidence/variants_limits.txt b/evidence/variants_limits.txt new file mode 100644 index 0000000..158a654 --- /dev/null +++ b/evidence/variants_limits.txt @@ -0,0 +1,20 @@ +ok 320x200 -> thumbnail 256x160 +ok crop 128x128 +ok optimized WebP 320x200, no upscale +ok thumbnail/crop are JPEG +ok 40x10: thumbnail 40x10, crop 128x128 +ok 3000x2 -> 256x1 / 1280x1 (min 1px) +ok 1001x999 -> 256x255 (floor) +ok EXIF orientation applied: 400x200 (o=6) -> 128x256 +ok rotated content: red half on top after orientation 6 +ok EXIF removed from thumbnail/optimized +ok transparent PNG -> white JPEG thumbnail +ok WebP optimized keeps alpha +ok 16-bit PNG, CMYK JPEG, palette+transparency PNG are processed +ok pixel limit enforced before decode -> image_too_large (exit 4) +ok corrupted PNG -> invalid_image (exit 3) +ok 36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got 5 +ok 39.5 Mpx PNG (31 MB file) ok in 1.0s, peak child RSS 320 MiB +ok CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got -24 +ok 39.5 Mpx JPEG (7 MB file) ok in 0.3s, peak child RSS 337 MiB +RESULT: PASS diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..2991e9d --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,1823 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — наблюдаемость (lab6)", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": false, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "graphTooltip": 1, + "panels": [ + { + "id": 1, + "type": "row", + "title": "SLO (окно 30 мин) и алерты", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Доступность (SLO 99%)", + "description": "1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.99 + }, + { + "color": "green", + "value": 0.995 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:ratio30m", + "legendFormat": "Доступность (SLO 99%)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Латентность ≤500 ms (SLO 95%)", + "description": "Доля запросов в bucket le=0.5 s за 30 мин.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 4, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.95 + }, + { + "color": "green", + "value": 0.97 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_latency:ratio30m", + "legendFormat": "Латентность ≤500 ms (SLO 95%)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Error ratio 5xx (2m)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_error_ratio:rate2m", + "legendFormat": "Error ratio 5xx (2m)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Старейшее задание", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachments.process\"})", + "legendFormat": "Старейшее задание", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 6, + "type": "alertlist", + "title": "Алерты Prometheus (firing/pending)", + "gridPos": { + "x": 16, + "y": 1, + "w": 8, + "h": 8 + }, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "firing": true, + "pending": true, + "noData": false, + "normal": false, + "error": true + }, + "viewMode": "list", + "datasource": "Prometheus" + } + }, + { + "id": 7, + "type": "stat", + "title": "RPS API", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_requests:rate2m", + "legendFormat": "RPS API", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 8, + "type": "stat", + "title": "p95 API (5m)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 4, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_latency_p95:rate5m", + "legendFormat": "p95 API (5m)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 9, + "type": "stat", + "title": "Загрузка worker", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:worker_saturation:ratio", + "legendFormat": "Загрузка worker", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "Ожидают пул БД", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(db_client_connection_pending_requests)", + "legendFormat": "Ожидают пул БД", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 11, + "type": "row", + "title": "API (messenger-api, все реплики)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "RPS по маршруту и методу", + "description": "Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 10, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\"}[$__rate_interval]))", + "legendFormat": "{{http_request_method}} {{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "RPS по классу статуса", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 10, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "normal" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\"}[$__rate_interval]), \"status_class\", \"${1}xx\", \"http_response_status_code\", \"(\\\\d)..\")) ", + "legendFormat": "{{status_class}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Латентность API p50/p95/p99 (из histogram buckets)", + "description": "histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 18, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Доля ошибок 5xx", + "description": "Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 18, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\"}[$__rate_interval]))", + "legendFormat": "5xx / все", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "p95 по маршруту", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 26, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Запросы в обработке", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 26, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (http_server_active_requests{service_name=\"messenger-api\"})", + "legendFormat": "{{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 18, + "type": "row", + "title": "Очередь и worker", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 34, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 19, + "type": "timeseries", + "title": "Jobs по исходу (в сек.)", + "description": "succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 35, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "normal" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Длительность job (succeeded) p50/p95/p99", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 35, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "Глубина очереди (из PostgreSQL)", + "description": "max() по репликам API: каждая публикует одно и то же значение из БД.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 43, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (queue, state) (messenger_queue_depth)", + "legendFormat": "{{queue}} {{state}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "Возраст старейшей job", + "description": "От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 43, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachments.process\"})", + "legendFormat": "oldest age", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 23, + "type": "timeseries", + "title": "Время до ready p50/p95/p99", + "description": "messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 51, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 24, + "type": "timeseries", + "title": "Насыщение worker: занято / ёмкость", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 51, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_busy)", + "legendFormat": "busy", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "capacity", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:worker_saturation:ratio", + "legendFormat": "saturation", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 25, + "type": "row", + "title": "Ресурсы: пул БД, CPU, RAM", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 59, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 26, + "type": "timeseries", + "title": "Пул БД: used / idle / waiting", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 60, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name, db_client_connection_state) (db_client_connection_count)", + "legendFormat": "{{service_name}} {{db_client_connection_state}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name) (db_client_connection_pending_requests)", + "legendFormat": "{{service_name}} waiting", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name) (db_client_connection_max)", + "legendFormat": "{{service_name}} max", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 27, + "type": "timeseries", + "title": "CPU процессов (ядра)", + "description": "worker: self + завершённые дочерние процессы обработки изображений.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 60, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "legendFormat": "{{service_name}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(otelcol_process_cpu_seconds_total[$__rate_interval])", + "legendFormat": "otel-collector", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 28, + "type": "timeseries", + "title": "RSS процессов", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 68, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_memory_usage_bytes", + "legendFormat": "{{service_name}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "otelcol_process_memory_rss_bytes", + "legendFormat": "otel-collector", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 29, + "type": "timeseries", + "title": "Telemetry pipeline: принято / отправлено / ошибки (items/s)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 68, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", + "legendFormat": "accepted spans", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", + "legendFormat": "accepted log records", + "refId": "E", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", + "legendFormat": "accepted metric points", + "refId": "F", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))", + "legendFormat": "sent {{exporter}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", + "legendFormat": "send failed spans", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", + "legendFormat": "dropped spans (queue full)", + "refId": "G", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", + "legendFormat": "send failed logs", + "refId": "H", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", + "legendFormat": "dropped logs (queue full)", + "refId": "I", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "queue {{exporter}}", + "refId": "D", + "range": true, + "instant": false + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Логи и трассы", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 76, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 31, + "type": "logs", + "title": "Логи API и worker (warning/error)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 77, + "w": 12, + "h": 10 + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "{service_name=~\"messenger-api|messenger-worker\"} | severity_text=~\"WARN.*|ERROR\"" + } + ] + }, + { + "id": 32, + "type": "table", + "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 12, + "y": 77, + "w": 12, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "limit": 20, + "query": "{resource.service.name=~\"messenger-.*\" && (status=error || duration>500ms)}" + } + ] + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "templating": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..d6b09ae --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..1267692 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,63 @@ +# Grafana 11.5.2 datasources (fixed uids are referenced by the dashboard JSON). Grafana stores no signals itself. +# "$$" escapes "$" from Grafana's env-variable interpolation; $__file{...} reads the client TLS material. +# lab7: every datasource uses HTTPS, verifies the server against the lab root CA and authenticates with grafana-out. +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: https://prometheus:9090 + isDefault: true + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + timeInterval: 15s + prometheusType: Prometheus + secureJsonData: + tlsCACert: "$__file{/run/secrets/ca.crt}" + tlsClientCert: "$__file{/run/secrets/client.crt}" + tlsClientKey: "$__file{/run/secrets/client.key}" + - name: Loki + uid: loki + type: loki + access: proxy + url: https://loki:3100 + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + maxLines: 1000 + derivedFields: + # trace_id is OTLP structured metadata -> link "Open trace" to Tempo. + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: Open trace + secureJsonData: + tlsCACert: "$__file{/run/secrets/ca.crt}" + tlsClientCert: "$__file{/run/secrets/client.crt}" + tlsClientKey: "$__file{/run/secrets/client.key}" + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: https://tempo:3200 + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"' + nodeGraph: + enabled: true + secureJsonData: + tlsCACert: "$__file{/run/secrets/ca.crt}" + tlsClientCert: "$__file{/run/secrets/client.crt}" + tlsClientKey: "$__file{/run/secrets/client.key}" diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..af05f5b --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,48 @@ +# Loki 3.4.2 single binary: log store. Logs arrive by OTLP (https://loki:3100/otlp, mTLS) from the Collector. +# service.name -> stream label service_name; log attributes (event, code.*) and trace_id/span_id -> structured metadata. +auth_enabled: false +server: + http_listen_port: 3100 + # Internal gRPC of the single binary: loopback only (closed local hop, not reachable from the network). + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9096 + log_level: warn + # lab7: HTTPS with mandatory client certificates from the "Telemetry Clients CA" (collector-out pushes, + # grafana-out queries, operator evidence). No certificate / another CA -> TLS handshake refused. + http_tls_config: + cert_file: /run/secrets/server.crt + key_file: /run/secrets/server.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/telemetry-ca.crt +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + volume_enabled: true +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/minio/init.sh b/infra/minio/init.sh new file mode 100755 index 0000000..11b62b7 --- /dev/null +++ b/infra/minio/init.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Idempotent MinIO bootstrap over HTTPS (CA verified by mc). Root credentials are used only here. +# Two service users with prefix-scoped policies: messenger-api and messenger-worker (see policy-*.json). +set -eu +BUCKET="${S3_BUCKET:-messenger-attachments}" +mkdir -p "$MC_CONFIG_DIR/certs/CAs" +cp /run/secrets/ca.crt "$MC_CONFIG_DIR/certs/CAs/ca.crt" +until mc alias set local https://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null 2>&1; do + echo "waiting for minio"; sleep 2 +done +mc mb --ignore-existing "local/$BUCKET" >/dev/null +mc anonymous set none "local/$BUCKET" >/dev/null +for role in api worker; do + user="$(cat /run/secrets/s3_${role}_access_key)" + mc admin policy create local "messenger-$role" "/etc/minio-policy/policy-$role.json" >/dev/null + mc admin user add local "$user" "$(cat /run/secrets/s3_${role}_secret_key)" >/dev/null + mc admin policy attach local "messenger-$role" --user "$user" >/dev/null 2>&1 || true +done +echo "minio bootstrap complete: bucket=$BUCKET private, users messenger-api / messenger-worker with prefix policies" diff --git a/infra/minio/policy-api.json b/infra/minio/policy-api.json new file mode 100644 index 0000000..8200b9d --- /dev/null +++ b/infra/minio/policy-api.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToStagingAndCompensate", + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"] + }, + { + "Sid": "DownloadAndReconcileFinalObjects", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"] + }, + { + "Sid": "ProbeAndListForReconciler", + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/minio/policy-worker.json b/infra/minio/policy-worker.json new file mode 100644 index 0000000..a3f01a9 --- /dev/null +++ b/infra/minio/policy-worker.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadAndCleanStaging", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"] + }, + { + "Sid": "WriteFinalObjectsOnly", + "Effect": "Allow", + "Action": ["s3:PutObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"] + } + ] +} diff --git a/infra/nginx-health.sh b/infra/nginx-health.sh new file mode 100644 index 0000000..1c791ad --- /dev/null +++ b/infra/nginx-health.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -u + +SERVERS="${API_UPSTREAMS:-api-1 api-2}" +INTERVAL="${HEALTH_INTERVAL:-1}" +FALL="${HEALTH_FALL:-2}" +RISE="${HEALTH_RISE:-2}" +OUT=/etc/nginx/upstream/servers.conf + +mkdir -p "$(dirname "$OUT")" + +var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; } + +render() { + any_up=0 + for s in $SERVERS; do + eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1 + done + for s in $SERVERS; do + mark="" + if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then + mark=" down" + fi + echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;" + done +} + +for s in $SERVERS; do + v=$(var "$s") + eval "up_$v=1 ok_$v=0 bad_$v=0" +done +render > "$OUT" + +watch() { + pending=0 + while sleep "$INTERVAL"; do + changed=0 + for s in $SERVERS; do + v=$(var "$s") + # Readiness over the same mTLS as real traffic (LB identity, server verified against the lab CA). + if curl -sf -m 1 -o /dev/null --cacert /run/secrets/ca.crt --cert /run/secrets/client.crt \ + --key /run/secrets/client.key "https://$s:8080/health/ready" 2>/dev/null; then + eval "ok_$v=\$((ok_$v + 1)) bad_$v=0" + if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then + eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream" + fi + else + eval "bad_$v=\$((bad_$v + 1)) ok_$v=0" + if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then + eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down" + fi + fi + done + if [ "$changed" = 1 ]; then + render > "$OUT" + pending=1 + fi + if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then + pending=0 + fi + done +} + +watch & +exec nginx -g 'daemon off;' diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 0000000..1b3ce0b --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,95 @@ +# nginx 1.27 (lab7): the only business ingress. Client -> nginx: TLS (server cert SAN localhost/nginx). +# nginx -> every API replica: mTLS. nginx presents lb-client (identity lb.client.messenger) and verifies the replica +# certificate against the lab root CA with the shared SAN api.internal. The API itself checks the LB identity. +worker_processes 2; +pid /var/run/nginx.pid; + +events { + worker_connections 256; +} + +http { + resolver 127.0.0.11 valid=5s ipv6=off; + + upstream api { + zone api 64k; + include /etc/nginx/upstream/servers.conf; + keepalive 32; + keepalive_timeout 4s; + } + + log_format main '$remote_addr - $upstream_addr [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$upstream_response_time" "$request_time"'; + + access_log /var/log/nginx/access.log main; + + server { + listen 8443 ssl; + ssl_certificate /run/secrets/server.crt; + ssl_certificate_key /run/secrets/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:tls:5m; + add_header Strict-Transport-Security "max-age=31536000" always; + server_tokens off; + client_max_body_size 64k; + error_page 413 @payload_too_large; + + # Upstream mTLS (no fallback to plaintext, verification always on). + proxy_ssl_certificate /run/secrets/client.crt; + proxy_ssl_certificate_key /run/secrets/client.key; + proxy_ssl_trusted_certificate /run/secrets/ca.crt; + proxy_ssl_verify on; + proxy_ssl_verify_depth 2; + proxy_ssl_name api.internal; + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + location ~ ^/api/v1/chats/[^/]+/messages$ { + proxy_pass https://api; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + } + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + client_max_body_size 10m; + proxy_request_buffering off; + proxy_pass https://api; + proxy_next_upstream off; + } + + location @payload_too_large { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id lb always; + add_header Connection close always; + return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}'; + } + + location /health/ { + proxy_pass https://api; + proxy_next_upstream error timeout; + } + + location ~ ^/api/v1/attachments/ { + proxy_max_temp_file_size 0; + proxy_pass https://api; + proxy_next_upstream error timeout; + } + + location / { + proxy_pass https://api; + proxy_next_upstream error timeout http_502 http_503; + } + } +} diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 0850e23..b8d90fd 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -1,7 +1,42 @@ -# OpenBao — реализовать конфигурацию и политики +# OpenBao — конфигурация и действия оператора -Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key. +OpenBao 2.6.5 в **server mode** (не dev): хранилище `file` на volume `baodata`, TLS-listener `openbao:8200` с сертификатом от lab Root CA. Наружу порт не опубликован; оператор работает через `docker compose exec` и CLI по TLS на `127.0.0.1` внутри контейнера. -Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения. +| Файл | Назначение | +| --- | --- | +| `config.hcl` | listener (TLS ≥ 1.2, `tls_client_ca_file` — запрос клиентского сертификата для cert auth), storage `file`, `disable_mlock` | +| `policies/messenger-api.hcl` | `read` только `messenger/data/api` и `messenger/metadata/api` | +| `policies/messenger-worker.hcl` | `read` только `messenger/data/worker` и `messenger/metadata/worker` | +| `../../scripts/bao.sh` | init, unseal, bootstrap, seal, rotate-worker-db, status (ничего секретного не печатает) | -Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md. +## Аутентификация сервисов + +Метод **cert**: +- роль `api`: сертификат цепочкой до lab Root CA и CN ровно `api.client.messenger`, политика `messenger-api`; +- роль `worker`: то же для `worker.client.messenger`, политика `messenger-worker`; +- у обеих ролей `token_ttl=15m`, `token_max_ttl=1h`. + +Сервис получает только свой TLS-ключ (Docker secret) и логинится им (`app/openbao.py`). Root token, SecretID, `BAO_TOKEN` в окружении приложения не используются. Каждые 15 с токен продлевается или запрашивается заново, а новая версия KV-секрета передаётся в callback (ротация без рестарта). + +## Действия оператора + +```sh +sh scripts/bao.sh init # один раз: 3 доли unseal, порог 2, root token -> secrets/operator/ (0600); затем unseal +sh scripts/bao.sh bootstrap # идемпотентно: KV v2 `messenger/`, политики, роли cert auth, секреты сервисов +sh scripts/bao.sh status +sh scripts/bao.sh unseal # ПОСЛЕ КАЖДОГО рестарта OpenBao (он стартует запечатанным) +sh scripts/bao.sh seal # учебный инцидент +sh scripts/bao.sh rotate-worker-db # новый пароль роли app_worker: ALTER ROLE + новая версия messenger/worker +``` + +**После рестарта OpenBao:** +- уже работающие API и worker продолжают работать на полученных секретах; +- `scripts/bao.sh unseal` возвращает возможность входа; +- сервис, перезапущенный при запечатанном OpenBao, ждёт до `BAO_STARTUP_TIMEOUT_SECONDS` (120 с) и завершается с кодом 78, Docker перезапускает его до unseal; +- restart hook `scripts/restart.sh` делает unseal сам, потому что его запускает оператор. + +**Восстановление и компрометация:** +- утрата root token — `bao operator generate-root` с порогом долей; +- компрометация роли сервиса — `bao delete auth/cert/certs/` (новые логины невозможны), отзыв токенов по accessor, ротация секретов в KV, новый сертификат. + +**Учебный компромисс.** Все 3 доли и root token лежат у одного оператора на хосте стенда (`secrets/operator/`, вне Git и образов, 0600). В production доли раздаются разным людям, root token отзывается после bootstrap, а для autounseal используется KMS или HSM. diff --git a/infra/openbao/config.hcl b/infra/openbao/config.hcl new file mode 100644 index 0000000..60a3680 --- /dev/null +++ b/infra/openbao/config.hcl @@ -0,0 +1,21 @@ +# OpenBao 2.1.0 server mode (NOT dev mode): persistent file storage on the volume `baodata`, TLS listener. +# The listener asks for a client certificate (tls_client_ca_file) because services authenticate with the cert +# auth method; requests without a certificate are still possible for the operator CLI with a token. +ui = false +disable_mlock = true # container without IPC_LOCK; swap is not configured in the lab VM (documented trade-off) +api_addr = "https://openbao:8200" +# Single node: no cluster listener at all (2.6 opens 8201 by default; nothing should listen without a purpose). +disable_clustering = true +log_level = "info" + +storage "file" { + path = "/openbao/file" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_cert_file = "/run/secrets/openbao.crt" + tls_key_file = "/run/secrets/openbao.key" + tls_client_ca_file = "/run/secrets/ca.crt" + tls_min_version = "tls12" +} diff --git a/infra/openbao/policies/messenger-api.hcl b/infra/openbao/policies/messenger-api.hcl new file mode 100644 index 0000000..8020659 --- /dev/null +++ b/infra/openbao/policies/messenger-api.hcl @@ -0,0 +1,7 @@ +# API replicas: read ONLY their own KV v2 secret (data + metadata for version checks). Nothing else. +path "messenger/data/api" { + capabilities = ["read"] +} +path "messenger/metadata/api" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/messenger-worker.hcl b/infra/openbao/policies/messenger-worker.hcl new file mode 100644 index 0000000..ce14b46 --- /dev/null +++ b/infra/openbao/policies/messenger-worker.hcl @@ -0,0 +1,7 @@ +# Worker: read ONLY its own KV v2 secret. It cannot read the API secret (other DB role, S3 user, Valkey ACL). +path "messenger/data/worker" { + capabilities = ["read"] +} +path "messenger/metadata/worker" { + capabilities = ["read"] +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..f5182f3 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,111 @@ +# OpenTelemetry Collector (contrib 0.120.0): receiver/processor/exporter only, it stores nothing long-term. +# API, worker --OTLP/HTTPS:4318 (mTLS)--> collector --mTLS--> Prometheus (OTLP), Loki (OTLP), Tempo (OTLP/gRPC) +# lab7: the receiver trusts ONLY the "Telemetry Ingest CA" (client_ca_file): api-otel / worker-otel certificates. +# A certificate from the root CA (LB, api, worker identities) or no certificate cannot push telemetry. +# Outbound: the collector presents collector-out (Telemetry Clients CA) and verifies each backend's server cert. +extensions: + health_check: + endpoint: 127.0.0.1:13133 # loopback only (no Docker healthcheck: distroless image) + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 4194304 # 4 MiB per OTLP request + tls: + cert_file: /run/secrets/server.crt + key_file: /run/secrets/server.key + client_ca_file: /run/secrets/ingest-ca.crt + min_version: "1.2" + + prometheus/self: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 15s + static_configs: [{targets: ["127.0.0.1:8888"]}] + +processors: + # First in every pipeline: refuse new data (receivers answer 503 -> SDK retries/drops) before the container OOMs. + # Container limit is 256 MiB; soft limit = limit_mib - spike_limit_mib = 140 MiB. + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + # Defense in depth: even if an instrumentation adds them, these attributes never leave the collector. + # url.query would contain ?filename=...; headers could contain Authorization. + attributes/scrub: + actions: + - {key: url.query, action: delete} + - {key: url.full, action: delete} + - {key: http.target, action: delete} + - {key: http.url, action: delete} + - {key: http.request.header.authorization, action: delete} + - {key: http.request.header.cookie, action: delete} + - {key: db.query.parameters, action: delete} + +exporters: + # Every exporter verifies the backend against the lab root CA and authenticates as collector-out (mTLS). + otlphttp/prometheus: + endpoint: https://prometheus:9090/api/v1/otlp + tls: &client_tls + ca_file: /run/secrets/ca.crt + cert_file: /run/secrets/client.crt + key_file: /run/secrets/client.key + min_version: "1.2" + compression: gzip + timeout: 5s + sending_queue: {enabled: true, num_consumers: 2, queue_size: 100} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s} + otlphttp/loki: + endpoint: https://loki:3100/otlp + tls: *client_tls + compression: gzip + timeout: 5s + sending_queue: {enabled: true, num_consumers: 2, queue_size: 100} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s} + otlp/tempo: + endpoint: tempo:4317 + tls: *client_tls + compression: gzip + timeout: 5s + sending_queue: {enabled: true, num_consumers: 2, queue_size: 100} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s} + +service: + extensions: [health_check] + telemetry: + # Stable identity of the collector's own metrics (default is a random UUID per restart -> label churn). + resource: + service.instance.id: otel-collector + logs: + level: info + metrics: + level: detailed + # lab7: self-metrics are served on LOOPBACK only and scraped by the collector itself (receiver + # prometheus/self), then exported through the normal metrics pipeline over mTLS. Nothing is reachable from the + # network in plaintext. (Direct OTLP push of internal telemetry ignored client certificates in 0.120.0.) + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp, prometheus/self] + processors: [memory_limiter, batch] + exporters: [otlphttp/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlphttp/loki] diff --git a/infra/otel/profile-default.yaml b/infra/otel/profile-default.yaml new file mode 100644 index 0000000..a2fa84a --- /dev/null +++ b/infra/otel/profile-default.yaml @@ -0,0 +1,5 @@ +# Default profile: no extra processing. All spans that the SDK head-sampled are exported. +# Selected with COLLECTOR_PROFILE=default (see compose.yaml); merged over collector.yaml. +processors: + batch: + timeout: 2s diff --git a/infra/otel/profile-tail.yaml b/infra/otel/profile-tail.yaml new file mode 100644 index 0000000..1e38de5 --- /dev/null +++ b/infra/otel/profile-tail.yaml @@ -0,0 +1,31 @@ +# Track 1 "balanced" profile (COLLECTOR_PROFILE=tail): SDK keeps 100% head sampling, the collector decides per +# whole trace after decision_wait. Keeps: every trace with an error, every trace slower than 500 ms, every trace of +# the attachment pipeline (rare, async, valuable), and 10% of the remaining ordinary traffic. +# Metrics are computed in the SDK BEFORE any sampling, so RPS/error rate/percentiles are unaffected. +processors: + tail_sampling: + decision_wait: 10s + num_traces: 20000 + expected_new_traces_per_sec: 200 + decision_cache: + sampled_cache_size: 50000 + policies: + - name: keep-errors + type: status_code + status_code: {status_codes: [ERROR]} + - name: keep-slow + type: latency + latency: {threshold_ms: 500} + - name: keep-async-pipeline + type: string_attribute + string_attribute: {key: messaging.system, values: [rabbitmq]} + - name: baseline-10pct + type: probabilistic + probabilistic: {sampling_percentage: 10} + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, tail_sampling, batch] + exporters: [otlp/tempo] diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..55d5eb1 --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,8 @@ +# PostgreSQL client authentication (lab7). Network clients: TLS only (hostssl) + SCRAM-SHA-256; plaintext refused. +# "local" = Unix socket inside the container only (healthcheck, operator `docker compose exec psql`): whoever can +# exec into the container already controls it, so trust adds no new access path. +local all all trust +hostssl all all 0.0.0.0/0 scram-sha-256 +hostssl all all ::/0 scram-sha-256 +host all all 0.0.0.0/0 reject +host all all ::/0 reject diff --git a/infra/prometheus/health-client.yml b/infra/prometheus/health-client.yml new file mode 100644 index 0000000..9cfff2b --- /dev/null +++ b/infra/prometheus/health-client.yml @@ -0,0 +1,5 @@ +# Client side of the container healthcheck (promtool check healthy): verify the server, present the operator cert. +tls_config: + ca_file: /run/secrets/ca.crt + cert_file: /run/secrets/health.crt + key_file: /run/secrets/health.key diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..8187aaf --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,25 @@ +# Prometheus v3.2.1: metrics store. Metrics arrive by OTLP push (--web.enable-otlp-receiver, /api/v1/otlp). +# The web listener is HTTPS with mandatory client certificates (web.yml). +global: + scrape_interval: 15s + evaluation_interval: 15s + +otlp: + # job=service.name, instance=service.instance.id are set by Prometheus; these become extra labels. + promote_resource_attributes: + - service.name + - service.instance.id + - service.version + - deployment.environment.name + +storage: + tsdb: + # OTLP pushes from several SDKs/batches may arrive slightly out of order. + out_of_order_time_window: 10m + +rule_files: + - /etc/prometheus/rules/*.yml + +# lab7: no scrape jobs. Every signal arrives by OTLP push over mTLS (application metrics through the Collector, +# Collector self-metrics directly). A scrape of a plaintext metrics port would be an unauthenticated side channel. +scrape_configs: [] diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..aad2c6a --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,111 @@ +# Recording rules (SLI/SLO) and alert rules. Evaluated by Prometheus every 15 s; state (pending/firing/inactive) +# is visible in Prometheus /alerts, the API /api/v1/alerts and in Grafana (alert list panel). No external delivery. +# Runbooks: docs/runbooks.md (anchors in runbook_url). +# +# Label mapping (OTLP -> Prometheus 3): service.name -> service_name, service.instance.id -> instance, +# http.route -> http_route, http.response.status_code -> http_response_status_code, job.type -> job_type. +# HTTP SLI denominator: every business request of messenger-api (health probes are not instrumented). +# Numerator of errors: 5xx only. 4xx (wrong password, 404 of a foreign chat, 409) are client outcomes, not failures. +groups: + - name: messenger-sli + interval: 15s + rules: + - record: messenger:http_requests:rate2m + expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[2m])) + - record: messenger:http_errors_5xx:rate2m + expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[2m])) or vector(0) + - record: messenger:http_error_ratio:rate2m + expr: messenger:http_errors_5xx:rate2m / messenger:http_requests:rate2m + - record: messenger:http_latency_p95:rate5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[5m]))) + # SLO window: rolling 30 min (lab-sized; production would use 28-30 days with the same formulas). + # Availability SLO: 99% of business requests without 5xx. + - record: messenger:slo_availability:ratio30m + expr: | + 1 - ( + (sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[30m])) or vector(0)) + / sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m])) + ) + # Latency SLO: 95% of business requests complete in <= 500 ms (bucket le="0.5"). + - record: messenger:slo_latency:ratio30m + expr: | + sum(increase(http_server_request_duration_seconds_bucket{service_name="messenger-api",le="0.5"}[30m])) + / sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m])) + - record: messenger:worker_saturation:ratio + # OTLP-pushed series have no staleness markers: a recreated worker would stay in the denominator for the + # 5 min lookback. Only samples fresher than 30 s (3 export intervals) are counted. + expr: | + sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 30)) + / sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30)) + + - name: messenger-alerts + rules: + - alert: MessengerApiHighErrorRate + expr: messenger:http_error_ratio:rate2m > 0.05 and messenger:http_requests:rate2m > 0.2 + for: 1m + labels: + severity: page + annotations: + summary: "API: доля 5xx {{ $value | humanizePercentage }} за 2 мин (порог 5%)" + description: "Больше 5% бизнес-запросов messenger-api завершаются 5xx (окно 2m, держится 1m). Расходуется бюджет SLO доступности 99%." + runbook_url: "docs/runbooks.md#messengerapihigherrorrate" + + - alert: MessengerQueueLagGrowing + expr: max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60 + for: 1m + labels: + severity: page + annotations: + summary: "Очередь вложений: старейшее задание ждёт {{ $value | humanizeDuration }}" + description: "Принятое задание не завершено дольше 60 с (порог), условие держится 1m: worker остановлен, перегружен или не может писать в S3/DB." + runbook_url: "docs/runbooks.md#messengerqueuelaggrowing" + + - alert: MessengerApiLatencyHigh + expr: messenger:http_latency_p95:rate5m > 1 + for: 2m + labels: + severity: ticket + annotations: + summary: "API p95 = {{ $value | humanizeDuration }} (порог 1 s, окно 5m)" + description: "p95 из histogram buckets всех реплик выше 1 s." + runbook_url: "docs/runbooks.md#messengerapilatencyhigh" + + - alert: MessengerDbPoolWaiting + expr: max(db_client_connection_pending_requests) > 0 + for: 1m + labels: + severity: ticket + annotations: + summary: "Запросы ждут соединение из пула БД ({{ $value }} в очереди)" + description: "Пул psycopg исчерпан дольше минуты: БД медленная/блокирована или пул мал." + runbook_url: "docs/runbooks.md#messengerdbpoolwaiting" + + - alert: MessengerWorkerSaturated + expr: messenger:worker_saturation:ratio > 0.9 + for: 5m + labels: + severity: ticket + annotations: + summary: "Worker занят на {{ $value | humanizePercentage }} ёмкости 5 минут" + runbook_url: "docs/runbooks.md#messengerworkersaturated" + + - alert: TelemetryExportFailing + # Each term has "or vector(0)": the *_failed_* series do not exist until the first failure, and a sum with an + # empty operand is empty (the alert would never fire). Early signal: exporter queue filling up (backend + # down, retries in progress); late signal: data dropped (queue full) or retries exhausted. + expr: | + ( + (sum(rate(otelcol_exporter_send_failed_spans_total[2m])) or vector(0)) + + (sum(rate(otelcol_exporter_send_failed_log_records_total[2m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_spans_total[2m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_log_records_total[2m])) or vector(0)) + ) > 0 + or + max(max by (exporter) (otelcol_exporter_queue_size) / max by (exporter) (otelcol_exporter_queue_capacity)) > 0.2 + for: 1m + labels: + severity: ticket + annotations: + summary: "Collector не может доставить telemetry в backend" + description: "Очередь exporter заполнена >20% или данные отбрасываются: Tempo/Loki/Prometheus недоступен. Бизнес-API не затронут, RAM collector ограничена очередью и memory_limiter." + runbook_url: "docs/runbooks.md#telemetryexportfailing" diff --git a/infra/prometheus/web.yml b/infra/prometheus/web.yml new file mode 100644 index 0000000..52b202d --- /dev/null +++ b/infra/prometheus/web.yml @@ -0,0 +1,9 @@ +# Prometheus web listener (lab7): TLS + mandatory client certificate from the "Telemetry Clients CA" only +# (collector-out for OTLP ingest, grafana-out for queries, operator for evidence scripts and the healthcheck). +# Root-signed service identities (api, worker, lb) are not in this CA and are refused at the handshake. +tls_server_config: + cert_file: /run/secrets/server.crt + key_file: /run/secrets/server.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/telemetry-ca.crt + min_version: TLS12 diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json new file mode 100644 index 0000000..a83b6b3 --- /dev/null +++ b/infra/rabbitmq/definitions.json @@ -0,0 +1,25 @@ +{ + "vhosts": [{"name": "messenger"}], + "users": [ + {"name": "api.client.messenger", "password_hash": "", "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []}, + {"name": "worker.client.messenger", "password_hash": "", "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} + ], + "permissions": [ + {"user": "api.client.messenger", "vhost": "messenger", "configure": "^$", "write": "^attachments$", "read": "^$"}, + {"user": "worker.client.messenger", "vhost": "messenger", "configure": "^$", "write": "^$", "read": "^attachments\\.process$"} + ], + "exchanges": [ + {"name": "attachments", "vhost": "messenger", "type": "direct", "durable": true, "auto_delete": false, "internal": false, "arguments": {}}, + {"name": "attachments.dlx", "vhost": "messenger", "type": "fanout", "durable": true, "auto_delete": false, "internal": false, "arguments": {}} + ], + "queues": [ + {"name": "attachments.process", "vhost": "messenger", "durable": true, "auto_delete": false, + "arguments": {"x-queue-type": "quorum", "x-dead-letter-exchange": "attachments.dlx", "x-delivery-limit": 20}}, + {"name": "attachments.dlq", "vhost": "messenger", "durable": true, "auto_delete": false, + "arguments": {"x-queue-type": "quorum"}} + ], + "bindings": [ + {"source": "attachments", "vhost": "messenger", "destination": "attachments.process", "destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}}, + {"source": "attachments.dlx", "vhost": "messenger", "destination": "attachments.dlq", "destination_type": "queue", "routing_key": "", "arguments": {}} + ] +} diff --git a/infra/rabbitmq/enabled_plugins b/infra/rabbitmq/enabled_plugins new file mode 100644 index 0000000..70d609f --- /dev/null +++ b/infra/rabbitmq/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_auth_mechanism_ssl]. diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..d5446d1 --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,25 @@ +# RabbitMQ 4.0 (lab7): TLS-only AMQP with mandatory client certificates. Identity = certificate CN, mapped to a +# passwordless user from definitions.json (auth mechanism EXTERNAL). No password users, no guest, no management UI. +listeners.tcp = none +listeners.ssl.default = 5671 +ssl_options.cacertfile = /run/secrets/ca.crt +ssl_options.certfile = /run/secrets/server.crt +ssl_options.keyfile = /run/secrets/server.key +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true +ssl_options.versions.1 = tlsv1.3 +ssl_options.versions.2 = tlsv1.2 +auth_mechanisms.1 = EXTERNAL +ssl_cert_login_from = common_name +# Users, permissions, vhost and topology come from the definitions file on every boot. +definitions.import_backend = local_filesystem +definitions.local.path = /etc/rabbitmq/definitions.json +definitions.skip_if_unchanged = true +loopback_users = none +consumer_timeout = 600000 +disk_free_limit.absolute = 200MB +vm_memory_high_watermark.relative = 0.6 +log.console.level = warning +# Erlang distribution (CLI tools, clustering) is loopback-only: no cluster here, and rabbitmqctl runs inside the +# container. Otherwise ports 25672/4369 would be a cookie-protected plaintext channel reachable from the network. +distribution.listener.interface = 127.0.0.1 diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..321e512 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,40 @@ +# Tempo 2.7.1 single binary: trace store (local blocks on a volume). Receives OTLP/gRPC (mTLS) from the Collector. +stream_over_http_enabled: true +server: + http_listen_port: 3200 + grpc_listen_address: 127.0.0.1 # internal gRPC of the single binary: loopback only + log_level: warn + # lab7: query API over HTTPS with mandatory client certificates from the "Telemetry Clients CA". + http_tls_config: + cert_file: /run/secrets/server.crt + key_file: /run/secrets/server.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/telemetry-ca.crt +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + # OTLP ingest from the Collector only: mTLS with the same client CA (collector-out). + tls: + cert_file: /run/secrets/server.crt + key_file: /run/secrets/server.key + client_ca_file: /run/secrets/telemetry-ca.crt +ingester: + max_block_duration: 5m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +query_frontend: + search: + max_result_limit: 10000 +usage_report: + reporting_enabled: false diff --git a/infra/valkey/valkey-start.sh b/infra/valkey/valkey-start.sh new file mode 100644 index 0000000..9939621 --- /dev/null +++ b/infra/valkey/valkey-start.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Valkey (lab7): TLS-only (plaintext port 0), ACL users rendered at start into the tmpfs from a Docker secret. +# default disabled +# api session cache + login rate limit keys only, a minimal command set (no FLUSH*, CONFIG, KEYS, SCAN, EVAL) +# health PING only (container healthcheck) +# Cache only: no persistence; PostgreSQL stays the source of truth for sessions. +set -eu +umask 077 +{ + echo "user default off" + echo "user api on >$(cat /run/secrets/valkey_api_password) resetkeys ~session:* ~ratelimit:* resetchannels -@all +get +set +del +incr +incrby +expire +ttl +ping +hello +client|setinfo +client|setname +select" + echo "user health on >health resetkeys -@all +ping" +} > /tmp/users.acl +exec valkey-server \ + --port 0 \ + --tls-port 6379 \ + --tls-cert-file /run/secrets/server.crt \ + --tls-key-file /run/secrets/server.key \ + --tls-ca-cert-file /run/secrets/ca.crt \ + --tls-auth-clients no \ + --aclfile /tmp/users.acl \ + --save "" --appendonly no \ + --maxmemory 64mb --maxmemory-policy allkeys-lru \ + --dir /data diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..4de361a --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,46 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name bytea NOT NULL + CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400), + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title bytea NOT NULL + CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400), + created_at timestamptz NOT NULL +); + +CREATE INDEX chats_created_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + text bytea NOT NULL + CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000), + created_at timestamptz NOT NULL +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 0000000..7981861 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + key text NOT NULL + CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'), + request_body_hash text NOT NULL, + response_json text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..1dc5563 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,28 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + filename text NOT NULL + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL + CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL + CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL, + status text NOT NULL DEFAULT 'ready' + CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/0004_allow_empty_text.sql b/migrations/0004_allow_empty_text.sql new file mode 100644 index 0000000..74aeab4 --- /dev/null +++ b/migrations/0004_allow_empty_text.sql @@ -0,0 +1,2 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_size; +ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000); diff --git a/migrations/0005_attachment_upload_state.sql b/migrations/0005_attachment_upload_state.sql new file mode 100644 index 0000000..f0489c5 --- /dev/null +++ b/migrations/0005_attachment_upload_state.sql @@ -0,0 +1,5 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed')); +ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT; +CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key); +CREATE INDEX attachments_status_created_idx ON attachments (status, created_at); diff --git a/migrations/0006_async_processing.sql b/migrations/0006_async_processing.sql new file mode 100644 index 0000000..64703d8 --- /dev/null +++ b/migrations/0006_async_processing.sql @@ -0,0 +1,75 @@ +-- lab5: upload -> staging -> job/outbox -> broker -> worker -> final prefix + variants. +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status + CHECK (status IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +-- staging_key: private S3 prefix written by the API before 202; object_key: final original written by the worker. +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +CREATE UNIQUE INDEX attachments_staging_key_idx ON attachments (staging_key) WHERE staging_key IS NOT NULL; + +-- Effect key of the pipeline: (attachment_id, pipeline_version, name). A repeated job cannot add a second set. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CONSTRAINT variants_ct CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CONSTRAINT variants_size CHECK (size_bytes > 0), + width integer NOT NULL CONSTRAINT variants_width CHECK (width > 0), + height integer NOT NULL CONSTRAINT variants_height CHECK (height > 0), + sha256 text NOT NULL CONSTRAINT variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- One logical job per (attachment, pipeline_version); job_id never changes on redelivery or retry. +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + state text NOT NULL + CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed')), + attempt integer NOT NULL DEFAULT 1 CONSTRAINT jobs_attempt CHECK (attempt >= 1), + max_attempts integer NOT NULL CONSTRAINT jobs_max_attempts CHECK (max_attempts >= 1), + next_attempt_at timestamptz NOT NULL, + lease_owner text, + lease_until timestamptz, + error_code text, + last_error text, + created_at timestamptz NOT NULL, + first_started_at timestamptz, + started_at timestamptz, + finished_at timestamptz, + updated_at timestamptz NOT NULL, + UNIQUE (attachment_id, pipeline_version) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE state = 'processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE state = 'failed'; +CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (state) WHERE state IN ('queued', 'processing'); + +-- Transactional outbox: written in the same transaction as the job; the relay publishes with confirms. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz, + publish_attempts integer NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL +); +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_published_idx ON outbox (published_at) WHERE published_at IS NOT NULL; + +-- Operator-visible history of every state change of a job (including explicit replay). +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL, + action text NOT NULL, + actor text NOT NULL, + attempt integer NOT NULL, + detail jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL +); +CREATE INDEX job_audit_job_idx ON job_audit (job_id, id); diff --git a/migrations/0007_service_roles.sql b/migrations/0007_service_roles.sql new file mode 100644 index 0000000..44f7ad2 --- /dev/null +++ b/migrations/0007_service_roles.sql @@ -0,0 +1,29 @@ +-- lab7: separate least-privilege DB roles for the services. Passwords are NOT here: scripts/migrate.py sets them +-- from operator bootstrap files and the services read them from OpenBao. The owner role `messenger` is only used +-- by migrations and the operator. +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'app_api') THEN + CREATE ROLE app_api LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT CONNECTION LIMIT 40; + END IF; + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'app_worker') THEN + CREATE ROLE app_worker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT CONNECTION LIMIT 20; + END IF; +END +$$; + +REVOKE ALL ON DATABASE messenger FROM PUBLIC; +GRANT CONNECT ON DATABASE messenger TO app_api, app_worker; +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO app_api, app_worker; + +-- API: business tables of every endpoint, the outbox relay, the reconciler and scripts/jobs.py. +GRANT SELECT, INSERT, UPDATE, DELETE ON users, chats, chat_members, messages, message_attachments, sessions, + idempotency_keys, attachments, attachment_variants, attachment_jobs, outbox, job_audit TO app_api; + +-- Worker: only the processing pipeline. No users/sessions/messages/chats, no DELETE anywhere. +GRANT SELECT, UPDATE ON attachments, attachment_jobs TO app_worker; +GRANT SELECT, INSERT ON attachment_variants, job_audit TO app_worker; +GRANT INSERT ON outbox TO app_worker; -- retry scheduling writes a new outbox row; publishing stays in the API + +GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO app_api, app_worker; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ed42b23 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,60 @@ +aio-pika==9.5.5 +aiormq==6.8.1 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +asgiref==3.13.0 +boto3==1.35.81 +botocore==1.35.81 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +Deprecated==3.0.1 +exceptiongroup==1.2.2 +fastapi==0.115.6 +googleapis-common-protos==1.75.0 +h11==0.16.0 +httptools==0.9.0 +idna==3.20 +importlib_metadata==8.5.0 +jmespath==1.0.1 +multidict==6.1.0 +opentelemetry-api==1.30.0 +opentelemetry-exporter-otlp-proto-common==1.30.0 +opentelemetry-exporter-otlp-proto-http==1.30.0 +opentelemetry-instrumentation==0.51b0 +opentelemetry-instrumentation-asgi==0.51b0 +opentelemetry-instrumentation-botocore==0.51b0 +opentelemetry-instrumentation-fastapi==0.51b0 +opentelemetry-propagator-aws-xray==1.0.2 +opentelemetry-proto==1.30.0 +opentelemetry-sdk==1.30.0 +opentelemetry-semantic-conventions==0.51b0 +opentelemetry-util-http==0.51b0 +packaging==26.3 +pamqp==3.3.0 +pillow==13.0.0 +propcache==0.2.1 +protobuf==5.29.6 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.11 +pydantic==2.14.1 +pydantic_core==2.50.1 +python-dateutil==2.9.0.post0 +requests==2.34.2 +s3transfer==0.10.4 +six==1.17.0 +starlette==0.41.3 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.0 +uvloop==0.23.0 +valkey==6.2.0 +wrapt==1.17.3 +yarl==1.18.3 +zipp==4.1.2 diff --git a/scripts/bao.sh b/scripts/bao.sh new file mode 100755 index 0000000..1690a7c --- /dev/null +++ b/scripts/bao.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# OpenBao operator tool. Unseal shares and the root token live ONLY in secrets/operator/ (git-ignored, 0600) and +# are never printed. Values are passed to the CLI through the environment of `docker compose exec`, not argv. +# +# scripts/bao.sh init initialise once: 3 unseal shares, threshold 2; then unseal +# scripts/bao.sh unseal unseal after every OpenBao restart (operator action; the API does not do it) +# scripts/bao.sh bootstrap KV v2 mount, policies, cert auth roles, service secrets (idempotent) +# scripts/bao.sh status sealed/initialised state +# scripts/bao.sh rotate-worker-db new password for DB role app_worker: PostgreSQL + OpenBao (level 4) +# scripts/bao.sh seal seal OpenBao (incident drill: services keep cached secrets, new ones must wait) +# +# Production note: each share would go to a different person; here one operator holds them on one host. +set -eu +cd "$(dirname "$0")/.." +OP=secrets/operator +BS=secrets/bootstrap +ENVS="-e BAO_ADDR=https://127.0.0.1:8200 -e BAO_CACERT=/run/secrets/ca.crt" + +bao() { docker compose exec -T $ENVS openbao bao "$@"; } +bao_root() { BAO_TOKEN="$(cat "$OP/root-token")" docker compose exec -T $ENVS -e BAO_TOKEN openbao bao "$@"; } + +wait_up() { + i=0 + until docker compose exec -T $ENVS openbao sh -c 'bao status >/dev/null 2>&1; [ $? -ne 1 ]'; do + i=$((i + 1)); [ $i -gt 60 ] && { echo "openbao not reachable" >&2; exit 1; }; sleep 1 + done +} + +sealed() { ! docker compose exec -T $ENVS openbao bao status >/dev/null 2>&1; } + +unseal() { + wait_up + if ! sealed; then echo "openbao: already unsealed"; return 0; fi + for n in 1 2; do + UNSEAL_KEY="$(cat "$OP/unseal-$n")" docker compose exec -T $ENVS -e UNSEAL_KEY openbao \ + sh -c 'bao operator unseal "$UNSEAL_KEY" >/dev/null' + done + sealed && { echo "openbao: still sealed" >&2; exit 1; } + echo "openbao: unsealed with 2 of 3 operator shares" +} + +init() { + wait_up + if bao status -format=json 2>/dev/null | python3 -c 'import json,sys;sys.exit(0 if json.load(sys.stdin)["initialized"] else 1)'; then + echo "openbao: already initialised"; unseal; return 0 + fi + umask 077; mkdir -p "$OP" + bao operator init -key-shares=3 -key-threshold=2 -format=json | python3 -c ' +import json, sys +d = json.load(sys.stdin) +for i, key in enumerate(d["unseal_keys_b64"], 1): + open("'"$OP"'/unseal-%d" % i, "w").write(key) +open("'"$OP"'/root-token", "w").write(d["root_token"]) +' + chmod 600 "$OP"/* + echo "openbao: initialised; 3 shares and root token stored in $OP/ (0600, not printed)" + unseal +} + +bootstrap() { + unseal + bao_root secrets list -format=json | grep -q '"messenger/"' || bao_root secrets enable -path=messenger -version=2 kv + bao_root auth list -format=json | grep -q '"cert/"' || bao_root auth enable cert + for p in messenger-api messenger-worker; do + docker compose exec -T openbao sh -c "cat > /tmp/$p.hcl" < "infra/openbao/policies/$p.hcl" + bao_root policy write "$p" "/tmp/$p.hcl" >/dev/null + done + docker compose exec -T openbao sh -c 'cp /run/secrets/ca.crt /tmp/ca.crt' + # Identity = client certificate chained to the lab root CA AND an exact common name. Short-lived tokens. + bao_root write auth/cert/certs/api certificate=@/tmp/ca.crt allowed_common_names=api.client.messenger \ + token_policies=messenger-api token_ttl=15m token_max_ttl=1h >/dev/null + bao_root write auth/cert/certs/worker certificate=@/tmp/ca.crt allowed_common_names=worker.client.messenger \ + token_policies=messenger-worker token_ttl=15m token_max_ttl=1h >/dev/null + kv_api + kv_worker + echo "openbao: mount messenger/ (kv v2), policies messenger-api/worker, cert roles api/worker, secrets written" +} + +# KV values are read from secrets/bootstrap and passed through stdin as JSON (never on a command line). +kv_api() { + python3 - "$BS" <<'PY' | bao_root kv put -mount=messenger api - >/dev/null +import json, sys +b = sys.argv[1] +r = lambda n: open("%s/%s" % (b, n)).read().strip() +print(json.dumps({ + "database_url": "postgresql://app_api:%s@postgres:5432/messenger" % r("db_api_password"), + "s3_access_key_id": r("s3_api_access_key"), "s3_secret_access_key": r("s3_api_secret_key"), + "cache_url": "rediss://api:%s@valkey:6379/0" % r("valkey_api_password")})) +PY +} + +kv_worker() { + python3 - "$BS" <<'PY' | bao_root kv put -mount=messenger worker - >/dev/null +import json, sys +b = sys.argv[1] +r = lambda n: open("%s/%s" % (b, n)).read().strip() +print(json.dumps({ + "database_url": "postgresql://app_worker:%s@postgres:5432/messenger" % r("db_worker_password"), + "s3_access_key_id": r("s3_worker_access_key"), "s3_secret_access_key": r("s3_worker_secret_key")})) +PY +} + +rotate_worker_db() { + unseal + umask 077 + python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > "$BS/db_worker_password.new" + chmod 644 "$BS/db_worker_password.new" + # 1) PostgreSQL first (open connections keep working), 2) OpenBao: the worker picks up the new version. + NEWPW="$(cat "$BS/db_worker_password.new")" docker compose exec -T -e NEWPW postgres sh -c \ + 'psql -v ON_ERROR_STOP=1 -q -U messenger -d messenger -v pw="$NEWPW" </dev/null; echo "openbao: sealed" ;; + status) wait_up; bao status | grep -E "Initialized|Sealed|Version|Storage" ;; + *) echo "usage: $0 init|unseal|bootstrap|status|rotate-worker-db|seal" >&2; exit 2 ;; +esac diff --git a/scripts/gen_dashboard.py b/scripts/gen_dashboard.py new file mode 100644 index 0000000..4c742dc --- /dev/null +++ b/scripts/gen_dashboard.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup). + +The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py +Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов"). +""" +import json +from pathlib import Path + +OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json" +PROM = {"type": "prometheus", "uid": "prometheus"} +LOKI = {"type": "loki", "uid": "loki"} +TEMPO = {"type": "tempo", "uid": "tempo"} +API = 'service_name="messenger-api"' +HTTP = "http_server_request_duration_seconds" +_ids = iter(range(1, 1000)) + + +def target(expr, legend="", ref="A", instant=False): + return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant, + "instant": instant} + + +def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None): + panel = { + "id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM, + "gridPos": {"x": x, "y": y, "w": w, "h": h}, + "fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, + "stacking": {"mode": "normal" if stack else "none"}, + "showPoints": "never"}}, + "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, + "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [target(*t) if isinstance(t, tuple) else t for t in targets], + } + if extra: + panel["fieldConfig"]["defaults"].update(extra) + return panel + + +def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None): + steps = thresholds or [{"color": "green", "value": None}] + panel = { + "id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM, + "gridPos": {"x": x, "y": y, "w": w, "h": h}, + "fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}}, + "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "colorMode": "value", "graphMode": "area", "textMode": "auto"}, + "targets": [target(expr, title)], + } + if decimals is not None: + panel["fieldConfig"]["defaults"]["decimals"] = decimals + return panel + + +def row(title, y): + return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, + "panels": []} + + +def q(p, expr_inner): + return "histogram_quantile(%s, %s)" % (p, expr_inner) + + +panels = [] +y = 0 +# ------------------------------------------------------------------ SLO and alerts +panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1 +red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad}, + {"color": "green", "value": warn}] +panels += [ + stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y, + desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.", + thresholds=red_yellow(0.99, 0.995), decimals=2), + stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y, + desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2), + stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2), + stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]), + {"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)", + "gridPos": {"x": 16, "y": y, "w": 8, "h": 8}, + "options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [], + "groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter": + {"firing": True, "pending": True, "noData": False, "normal": False, "error": True}, + "viewMode": "list", "datasource": "Prometheus"}}, +] +y += 4 +panels += [ + stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1), + stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]), + stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]), + stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]), +] +y += 4 +# ------------------------------------------------------------------ API +panels.append(row("API (messenger-api, все реплики)", y)); y += 1 +rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API) +panels += [ + ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count, + "{{http_request_method}} {{http_route}}")], "reqps", 0, y, + desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."), + ts("RPS по классу статуса", [( + 'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count, + "{{status_class}}")], "reqps", 12, y, stack=True), +] +y += 8 +bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API) +panels += [ + ts("Латентность API p50/p95/p99 (из histogram buckets)", [ + (q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y, + desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."), + ts("Доля ошибок 5xx", [ + ('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count), + "5xx / все", "A")], "percentunit", 12, y, + desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."), +] +y += 8 +panels += [ + ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)), + "{{http_route}}")], "s", 0, y), + ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], + "short", 12, y), +] +y += 8 +# ------------------------------------------------------------------ worker and queue +panels.append(row("Очередь и worker", y)); y += 1 +jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))" +rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))" +panels += [ + ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", + "{{outcome}}")], "ops", 0, y, stack=True, + desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"), + ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"), + (q(0.99, jb), "p99", "C")], "s", 12, y), +] +y += 8 +panels += [ + ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")], + "short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."), + ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")], + "s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.", + extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never", + "thresholdsStyle": {"mode": "line"}}, + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, + {"color": "red", "value": 60}]}}), +] +y += 8 +panels += [ + ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")], + "s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."), + ts("Насыщение worker: занято / ёмкость", [ + ("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"), + ("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y), +] +y += 8 +# ------------------------------------------------------------------ resources +panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1 +panels += [ + ts("Пул БД: used / idle / waiting", [ + ("sum by (service_name, db_client_connection_state) (db_client_connection_count)", + "{{service_name}} {{db_client_connection_state}}", "A"), + ("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"), + ("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y), + ts("CPU процессов (ядра)", [ + ("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "{{service_name}} {{instance}}", "A"), + ("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y, + desc="worker: self + завершённые дочерние процессы обработки изображений."), +] +y += 8 +panels += [ + ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"), + ("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y), + ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [ + ("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"), + ("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"), + ("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"), + ("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))", + "sent {{exporter}}", "B"), + # Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist. + ("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"), + ("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"), + ("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"), + ("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"), + ("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y), +] +y += 8 +# ------------------------------------------------------------------ logs and traces +panels.append(row("Логи и трассы", y)); y += 1 +panels += [ + {"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI, + "gridPos": {"x": 0, "y": y, "w": 12, "h": 10}, + "options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}, + "targets": [{"datasource": LOKI, "refId": "A", + "expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]}, + {"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO, + "gridPos": {"x": 12, "y": y, "w": 12, "h": 10}, + "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, + "query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]}, +] +y += 10 + +dashboard = { + "uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"], + "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s", + "time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels, + "annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, + "enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]}, + "templating": {"list": []}, +} +OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels))) + +# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script). +UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки", + "bytes": "байты"} +DOC = OUT.parents[3] / "docs/dashboard.md" +lines = ["# Панели dashboard «%s»" % dashboard["title"], "", + "Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. " + "`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""] +for panel in panels: + if panel["type"] == "row": + lines += ["## " + panel["title"], ""] + continue + unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "") + lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else "")) + if panel.get("description"): + lines.append("— " + panel["description"]) + lines.append("") + if panel["type"] == "alertlist": + lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""] + for t in panel.get("targets", []): + query = t.get("expr") or t.get("query") + legend = t.get("legendFormat", "") + lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""), + query + (" # " + legend if legend and legend != panel["title"] else ""), "```"] + lines.append("") +DOC.write_text("\n".join(lines), encoding="utf-8") +print("wrote docs/dashboard.md") diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 0000000..905f792 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Operator step 1 (after scripts/pki.sh init): local .env and random bootstrap credentials in secrets/bootstrap/. +# Nothing is printed. Existing files are kept, so a re-run never changes the passwords of a live stand. +# +# Who gets what: +# stores (postgres, minio, valkey, grafana) read their OWN admin/ACL material from Docker secrets; +# API and worker get NO password files at all: scripts/bao.sh bootstrap copies their credentials into OpenBao +# and the services read them there after authenticating with their client certificate. +set -eu +cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env +umask 077 +mkdir -p secrets/bootstrap +cd secrets/bootstrap + +put() { [ -f "$1" ] || printf '%s' "$2" > "$1"; } +random() { python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')"; } + +put postgres_password "$(random)" # superuser `messenger`: migrations and operator only +put db_api_password "$(random)" # role app_api +put db_worker_password "$(random)" # role app_worker +put minio_root_user messenger_root +put minio_root_password "$(random)" +put s3_api_access_key messenger-api +put s3_api_secret_key "$(random)" +put s3_worker_access_key messenger-worker +put s3_worker_secret_key "$(random)" +put valkey_api_password "$(random)" # ACL user `api` +put grafana_admin_password "$(random)" +put migrate_database_url "postgresql://messenger:$(cat postgres_password)@postgres:5432/messenger" +chmod 644 ./* # read by container users through Docker secrets; the directory itself is 700 +cd .. +chmod 700 . bootstrap +echo "bootstrap credentials ready in secrets/bootstrap/ (.env present)" diff --git a/scripts/jobs.py b/scripts/jobs.py new file mode 100755 index 0000000..91edc79 --- /dev/null +++ b/scripts/jobs.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...). + + failed registry of failed jobs (the "DLQ" of business failures) + show job state + audit trail + replay --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing + stats counts by state and outbox backlog +Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md). +""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import jobs # noqa: E402 +from app.cursor import fmt # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +def out(value) -> None: + print(json.dumps(value, ensure_ascii=False, indent=2, default=str)) + + +async def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("failed") + sub.add_parser("stats") + show = sub.add_parser("show") + show.add_argument("attachment_id") + rp = sub.add_parser("replay") + rp.add_argument("attachment_id") + rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)") + args = parser.parse_args() + + db = Database() + await db.open() + try: + if args.cmd == "failed": + async with db.connection() as conn: + out(await jobs.failed_jobs(conn)) + elif args.cmd == "stats": + async with db.connection() as conn: + cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state") + states = {r["state"]: r["n"] for r in await cur.fetchall()} + cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL") + backlog = (await cur.fetchone())["n"] + out({"jobs": states, "outbox_unpublished": backlog}) + elif args.cmd == "show": + async with db.connection() as conn: + cur = await conn.execute( + "SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until," + " a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.attachment_id = %s", (args.attachment_id,)) + job = await cur.fetchone() + cur = await conn.execute( + "SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id", + (args.attachment_id,)) + trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()] + cur = await conn.execute( + "SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts" + " FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id", + (args.attachment_id,)) + outbox = await cur.fetchall() + out({"job": job, "audit": trail, "outbox": outbox}) + elif args.cmd == "replay": + s3 = S3Storage() + await run_in_threadpool(s3.open) + + async def exists(key): + return await run_in_threadpool(s3.exists, key) + + result = await db.in_transaction( + lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists)) + out(result) + return 0 if result["result"] == "queued" else 3 + finally: + await db.close() + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 0000000..7d0962a --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run.""" +import hashlib +import os +import pathlib +import sys + +import psycopg + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MIGRATIONS = ROOT / "migrations" +LOCK_ID = 2026_0004 + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return pathlib.Path(path).read_text(encoding="utf-8").strip() + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + sys.exit("DATABASE_URL or DATABASE_URL_FILE is required") + return url + + +def set_service_passwords(conn) -> None: + """lab7: passwords of the service roles come from operator bootstrap files (never from Git or the image). + Re-running keeps them in sync with secrets/bootstrap, which scripts/bao.sh also copies into OpenBao.""" + from psycopg import sql + + for role, env in (("app_api", "DB_API_PASSWORD_FILE"), ("app_worker", "DB_WORKER_PASSWORD_FILE")): + path = os.environ.get(env) + if not path: + continue + with open(path, encoding="utf-8") as handle: + password = handle.read().strip() + conn.execute(sql.SQL("ALTER ROLE {} PASSWORD {}").format(sql.Identifier(role), sql.Literal(password))) + print("password set for role %s" % role) + + +def main() -> None: + files = sorted(MIGRATIONS.glob("*.sql")) + if not files: + sys.exit("no migration files found in %s" % MIGRATIONS) + tls = {"sslmode": os.environ.get("DATABASE_SSLMODE", "verify-full"), + "sslrootcert": os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt")} + with psycopg.connect(database_url(), autocommit=True, connect_timeout=10, **tls) as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall() + applied = {row[0]: row[1] for row in rows} + for path in files: + version = path.stem + body = path.read_text(encoding="utf-8") + checksum = hashlib.sha256(body.encode("utf-8")).hexdigest() + if version in applied: + if applied[version] != checksum: + sys.exit("migration %s changed after it was applied" % version) + print("skip %s" % version) + continue + with conn.transaction(): + conn.execute(body) + conn.execute( + "INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", + (version, checksum), + ) + print("apply %s" % version) + set_service_passwords(conn) + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + print("migrations complete: %d file(s)" % len(files)) + + +if __name__ == "__main__": + main() diff --git a/scripts/pki.sh b/scripts/pki.sh new file mode 100755 index 0000000..20dd002 --- /dev/null +++ b/scripts/pki.sh @@ -0,0 +1,190 @@ +#!/bin/sh +# Local lab PKI (operator tool). Private keys stay in secrets/pki/ (git-ignored, never baked into images). +# +# scripts/pki.sh init create CAs and every leaf that does not exist yet (idempotent) +# scripts/pki.sh issue (re)issue one leaf with a new key and serial (rotation) +# scripts/pki.sh revoke add the CURRENT serial of to secrets/pki/revoked-serials.txt +# scripts/pki.sh revoke-serial revoke a serial that is no longer the current one (after a rotation) +# scripts/pki.sh expired test-only: an ALREADY EXPIRED copy of (same identity) -> -expired.* +# scripts/pki.sh serial current serial of +# scripts/pki.sh list leaf names, identities, serials and expiry (no key material) +# +# Trust layout (identity is checked by every server, not only the chain): +# root CA "Messenger Lab Root CA" signs every server certificate and the service client identities +# (lb, api, worker) used for LB->API, RabbitMQ (EXTERNAL = certificate CN) and OpenBao cert auth. +# ingest CA (intermediate) signs ONLY api-otel / worker-otel: the Collector trusts nothing else, so a +# root-signed certificate (e.g. the LB one) cannot push telemetry. +# telemetry CA (intermediate) signs ONLY collector-out / grafana-out / operator: Prometheus, Loki and Tempo +# accept only these clients. +# intruder root-signed, valid, but with an identity that no server allows (negative tests). +# foreign CA unrelated CA; foreign-client claims the lb/api/worker names (negative tests of chain checks). +# OpenSSL runs in a pinned container, so the host OpenSSL/LibreSSL version does not matter. +set -eu +cd "$(dirname "$0")/.." +DIR=secrets/pki +IMAGE="alpine/openssl:3.3.2@sha256:5008e829163320a6e8166883c03e68189e8925ade68cde36584dc2a41cfa5248" +LEAF_DAYS="${LEAF_DAYS:-90}" +CA_DAYS=825 + +# name | signer | usage | CN | DNS/IP SANs +LEAVES=' +lb-server root server localhost localhost,nginx,IP:127.0.0.1 +lb-client root client lb.client.messenger lb.client.messenger +api-1 root server api-1 api-1,api.internal +api-2 root server api-2 api-2,api.internal +api root client api.client.messenger api.client.messenger +worker root client worker.client.messenger worker.client.messenger +postgres root server postgres postgres +minio root server minio minio +rabbitmq root server rabbitmq rabbitmq +valkey root server valkey valkey +openbao root server openbao openbao,localhost,IP:127.0.0.1 +otel-collector root server otel-collector otel-collector +prometheus root server prometheus prometheus,localhost,IP:127.0.0.1 +loki root server loki loki,localhost,IP:127.0.0.1 +tempo root server tempo tempo,localhost,IP:127.0.0.1 +grafana root server grafana grafana,localhost,IP:127.0.0.1 +api-otel ingest client api.otel.messenger api.otel.messenger +worker-otel ingest client worker.otel.messenger worker.otel.messenger +collector-out telemetry client collector.out.messenger collector.out.messenger +grafana-out telemetry client grafana.out.messenger grafana.out.messenger +operator telemetry client operator.messenger operator.messenger +intruder root client intruder.client.messenger intruder.client.messenger +' + +ossl() { # quiet on success, the OpenSSL error on failure + out=$(docker run --rm -u "$(id -u):$(id -g)" -v "$PWD/$DIR:/pki" -w /pki "$IMAGE" "$@" 2>&1) || { + echo "openssl $1 failed: $out" >&2; exit 1; } +} + +ca_init() { # ca_init [parent] + [ -f "$DIR/$1.crt" ] && return 0 + cat > "$DIR/.ext-$1" < + line=$(leaf_line "$1") + [ -n "$line" ] || { echo "unknown leaf: $1" >&2; exit 2; } + set -- $line + name=$1 signer=$2 usage=$3 cn=$4 sans=$5 + eku=serverAuth; [ "$usage" = client ] && eku=clientAuth + alt=$(echo "$sans" | tr ',' '\n' | awk '/^IP:/ {print; next} {print "DNS:" $0}' | paste -sd, -) + cat > "$DIR/.ext-$name" <): the files are bind-mounted Docker secrets, a new inode would not be visible. + if [ "$signer" = root ]; then cat "$DIR/$name.crt.new" > "$DIR/$name.crt"; else cat "$DIR/$name.crt.new" "$DIR/$signer.crt" > "$DIR/$name.crt"; fi + cat "$DIR/$name.key.new" > "$DIR/$name.key"; rm -f "$DIR/$name.key.new" + rm -f "$DIR/$name.crt.new" "$DIR/$name.csr" "$DIR/.ext-$name" + echo "issued $name ($cn) serial=$(serial "$name") valid ${LEAF_DAYS}d" +} + +foreign() { # an unrelated CA and a leaf that CLAIMS the LB/api/worker names: negative tests of chain checks + [ -f "$DIR/foreign-client.crt" ] && return 0 + ca_init foreign "/O=Somebody Else/CN=Foreign CA" + printf '%s\n' "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature,keyEncipherment" \ + "extendedKeyUsage=clientAuth,serverAuth" \ + "subjectAltName=DNS:lb.client.messenger,DNS:api.client.messenger,DNS:worker.client.messenger" \ + > "$DIR/.ext-foreign-client" + ossl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out foreign-client.key + ossl req -new -key foreign-client.key -subj "/O=Somebody Else/CN=lb.client.messenger" -out foreign-client.csr + ossl x509 -req -in foreign-client.csr -CA foreign.crt -CAkey foreign.key -CAcreateserial -days 30 -sha256 \ + -extfile .ext-foreign-client -out foreign-client.crt + rm -f "$DIR/foreign-client.csr" "$DIR/.ext-foreign-client" + echo "created foreign CA + foreign-client (claims lb/api/worker identities, trusted nowhere)" +} + +expired() { # same CN/SAN/EKU as , validity in the past (openssl ca allows explicit dates) + line=$(leaf_line "$1"); [ -n "$line" ] || { echo "unknown leaf: $1" >&2; exit 2; } + set -- $line + name=$1 signer=$2 usage=$3 cn=$4 sans=$5 + eku=serverAuth; [ "$usage" = client ] && eku=clientAuth + alt=$(echo "$sans" | tr ',' '\n' | awk '/^IP:/ {print; next} {print "DNS:" $0}' | paste -sd, -) + t="$DIR/.ca-$name"; rm -rf "$t"; mkdir -p "$t"; touch "$t/index.txt"; echo 01 > "$t/serial" + printf '%s\n' "[ca]" "default_ca=x" "[x]" "database=/pki/.ca-$name/index.txt" "serial=/pki/.ca-$name/serial" \ + "new_certs_dir=/pki/.ca-$name" "default_md=sha256" "policy=p" "unique_subject=no" "[p]" "commonName=supplied" \ + "[ext]" "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" "extendedKeyUsage=$eku" \ + "subjectAltName=$alt" > "$t/ca.cnf" + ossl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$name-expired.key" + ossl req -new -key "$name-expired.key" -subj "/O=Messenger Lab/CN=$cn" -out ".ca-$name/req.csr" + ossl ca -batch -config ".ca-$name/ca.cnf" -extensions ext -cert "$signer.crt" -keyfile "$signer.key" \ + -startdate 20250101000000Z -enddate 20250102000000Z -in ".ca-$name/req.csr" -out "$name-expired.crt" -notext + rm -rf "$t" + echo "issued $name-expired ($cn), validity 2025-01-01..2025-01-02 (already expired, test only)" +} + +serial() { docker run --rm -v "$PWD/$DIR:/pki" -w /pki "$IMAGE" x509 -in "$1.crt" -noout -serial | cut -d= -f2; } + +finish() { + cp "$DIR/root.crt" "$DIR/ca.crt" # public CA bundle for clients (make test CA_FILE=...) + cat "$DIR/root.crt" "$DIR/ingest.crt" > "$DIR/ingest-bundle.crt" + cat "$DIR/root.crt" "$DIR/telemetry.crt" > "$DIR/telemetry-bundle.crt" + touch "$DIR/revoked-serials.txt" + # Containers run as other uids and read keys through Docker secrets: files readable, directory closed. + chmod 700 secrets "$DIR" + chmod 644 "$DIR"/*.crt "$DIR"/*.key "$DIR/revoked-serials.txt" + chmod 600 "$DIR"/*.previous 2>/dev/null || true + chmod 600 "$DIR"/root.key "$DIR"/ingest.key "$DIR"/telemetry.key "$DIR"/foreign.key +} + +cmd="${1:-init}" +mkdir -p "$DIR" +case "$cmd" in + init) + ca_init root "/O=Messenger Lab/CN=Messenger Lab Root CA" + ca_init ingest "/O=Messenger Lab/CN=Messenger Telemetry Ingest CA" root + ca_init telemetry "/O=Messenger Lab/CN=Messenger Telemetry Clients CA" root + for name in $(echo "$LEAVES" | awk 'NF {print $1}'); do + [ -f "$DIR/$name.crt" ] || issue "$name" + done + foreign + finish ;; + issue) issue "$2"; finish ;; + revoke) + s=$(serial "$2"); grep -qx "$s" "$DIR/revoked-serials.txt" 2>/dev/null || echo "$s" >> "$DIR/revoked-serials.txt" + echo "revoked $2 serial=$s"; finish ;; + revoke-serial) + s=$(echo "$2" | tr 'a-f' 'A-F') + # append in place: the file is bind-mounted into the API containers (a new inode would not be seen) + grep -qx "$s" "$DIR/revoked-serials.txt" 2>/dev/null || echo "$s" >> "$DIR/revoked-serials.txt" + echo "revoked serial=$s" ;; + expired) expired "$2"; finish ;; + serial) serial "$2" ;; + list) + for name in $(echo "$LEAVES" | awk 'NF {print $1}'); do + printf '%-15s %-28s %s %s\n' "$name" "$(leaf_line "$name" | awk '{print $4}')" "$(serial "$name")" \ + "$(docker run --rm -v "$PWD/$DIR:/pki" -w /pki "$IMAGE" x509 -in "$name.crt" -noout -enddate | cut -d= -f2)" + done ;; + *) echo "usage: $0 init|issue |revoke |revoke-serial |expired |serial |list" >&2; exit 2 ;; +esac diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 0000000..10cd7b6 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything.""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import config, reconcile # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports") + parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS) + args = parser.parse_args() + db, s3 = Database(), S3Storage() + await db.open() + await run_in_threadpool(s3.open) + try: + print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True)) + finally: + await db.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..271171c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Restart hook (tests/resilience.py --mode persistence): restart every store keeping all volumes, then the services. +# business stores: postgres, valkey, minio, rabbitmq +# secret store: openbao -> comes back SEALED; the operator step `scripts/bao.sh unseal` uses 2 of 3 shares +# from secrets/operator/ (lab compromise documented in REPORT.md). Without it the services would +# wait for OpenBao after their own restart. +# telemetry: prometheus, loki, tempo, otel-collector, grafana +# services: worker, api-1, api-2, nginx (they log in to OpenBao again and re-read their secrets) +# Never removes volumes, never re-seeds. Prints no secrets. +set -eu +cd "$(dirname "$0")/.." +docker compose restart -t 10 postgres valkey minio rabbitmq openbao >/dev/null +sh scripts/bao.sh unseal >/dev/null +docker compose restart -t 10 prometheus loki tempo otel-collector grafana >/dev/null +docker compose restart -t 25 worker api-1 api-2 nginx >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..23c94b5 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +docker compose stop -t 3 "$TARGET_INSTANCE_ID" diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..2bbce62 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,116 @@ -# Модель угроз — заполнить +# Модель угроз — lab7 ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +**Активы:** +- пользовательские credentials: Argon2id-хеши паролей, Bearer-сессии (в БД — SHA-256 токена); +- сообщения и вложения: PostgreSQL и MinIO; +- service identities: TLS-ключи `lb`, `api`, `worker`, `api-otel`, `worker-otel`, `collector-out`, `grafana-out`, `operator`; +- закрытые ключи CA, root token и доли unseal OpenBao; +- секреты сервисов в OpenBao: пароли ролей БД, ключи S3, пароль Valkey; +- telemetry: логи, трассы, метрики. + +**Акторы:** + +| Актор | Возможности в модели | +| --- | --- | +| Участник чата | легитимный пользователь с Bearer-токеном | +| Посторонний пользователь | зарегистрирован, но не состоит в чате; знает или угадывает UUID чужих объектов | +| Внешний клиент | доступ только к `127.0.0.1:8443`, может перебирать пароли и подделывать заголовки | +| Скомпрометированный сервис | атакующий исполняет код в одном контейнере (worker, API, Grafana) и получает его ключи и сетевую позицию во внутренней сети | +| Посторонний контейнер в сети | любой процесс в сети Compose без выданных нами ключей, либо с сертификатом чужого CA | +| Оператор | доверенный: держит ключи CA, root token и доли unseal | + +**Границы стенда:** +- один Docker-host, сеть Compose считается враждебной: все связи идут по TLS; +- компрометация самого хоста и Docker daemon вне модели (root на хосте читает всё); +- данные синтетические. ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +``` +внешний клиент ──TLS──► nginx ──mTLS (lb identity, SAN allowlist)──► api-1, api-2 + │ TLS verify-full / HTTPS / AMQPS mTLS / TLS+ACL + worker ─────────────────────────────┼──► PostgreSQL, MinIO, RabbitMQ, Valkey + api, worker ──HTTPS + cert auth──► OpenBao (KV: свой путь, токен 15 мин) + api, worker ──mTLS (Ingest CA)──► Collector ──mTLS (Telemetry Clients CA)──► Prometheus / Loki / Tempo + Grafana ──mTLS (Telemetry Clients CA)──┘ +оператор ──docker compose exec / 127.0.0.1 (HTTPS, mTLS)──► OpenBao, Grafana, Prometheus/Loki/Tempo +``` -| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | -| --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | +Полная матрица со всеми портами, CA, identities и тестами: [contracts/trust-matrix.md](../contracts/trust-matrix.md). -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +## Угрозы, контроли и проверки + +| # | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | +| --- | --- | --- | --- | --- | --- | +| T1 | Скомпрометированный worker или посторонний контейнер подключается к `api-1:8080` напрямую и вызывает бизнес-API в обход nginx, подделывая `X-Real-IP` | к API ходит только LB; клиентский IP для лимитов достоверен | mTLS CERT_REQUIRED + allowlist DNS SAN `lb.client.messenger` + проверка отозванных serial в `app/serve.py`; plaintext-listener нет; health только на 127.0.0.1 | аудит A: без сертификата, чужой CA с именем lb, `worker`/`api`/`intruder` от того же CA → соединение закрыто до HTTP; аудит K: 8081 закрыт | скомпрометированный nginx имеет полный доступ к API (он и есть LB) | +| T2 | Скомпрометированный worker читает пользователей, сессии и сообщения в БД, удаляет вложения, публикует поддельные задания или читает секрет API | worker видит только конвейер обработки | роль `app_worker` (SELECT/UPDATE jobs и attachments, INSERT variants/audit/outbox, без DELETE); S3-пользователь worker: GET `staging/*`, PUT `attachments/*`; RabbitMQ: только read очереди; OpenBao: только `messenger/worker` | аудит B: SELECT users/messages/sessions и DELETE → InsufficientPrivilege; C: GET/DELETE `attachments`, LIST, PUT `staging` → AccessDenied; D: publish → ACCESS_REFUSED; H: чтение `messenger/api` → 403 | worker может записать любые байты в `attachments/*` (подмена вариантов) и испортить статус заданий; это минимум, нужный для его работы | +| T3 | Украден ключ LB-клиента: атакующий во внутренней сети представляется LB | украденная identity отзывается без пересборки и не принимается | короткий срок leaf (90 дн.), `pki.sh issue` + `nginx -s reload`, `pki.sh revoke-serial` → API отклоняет старый serial через ~1 с; просроченные сертификаты отклоняет TLS | `test_cert_rotation.py`: после отзыва старый сертификат закрыт на обеих репликах (`reason=revoked`), новый работает, просроченный отклонён; 0 ошибок трафика | до отзыва украденный ключ действует; отзыв проверяет только API | +| T4 | Скомпрометированный сервис с сертификатом от того же Root CA (например, LB или api) пишет поддельную telemetry напрямую в Prometheus/Loki/Tempo, читает чужие логи или обходит Collector | ingest только через Collector и только от приложений; хранилища читают только collector, grafana и оператор | отдельные промежуточные CA: Collector доверяет только Ingest CA, бэкенды — только Telemetry Clients CA | аудит E: `lb`, `api` (Root), `grafana-out` → отклонены Collector; аудит F: `api`, `api-otel` → отклонены Prometheus/Loki/Tempo | Collector не связывает `service.name` с сертификатом: `api-otel` может выдать себя за worker в telemetry | +| T5 | Внешний клиент перебирает пароли (credential stuffing) через обе реплики: каждая реплика считает попытки отдельно | ограничение действует на систему целиком | счётчики в Valkey, общие для реплик: 5 неудачных попыток на username и 120 попыток на IP за 60 с → 429 + Retry-After, до вычисления Argon2 | `test_auth_acl.py`: 5 неудач через `api-1` и `api-2` → 429 даже с верным паролем; 107 попыток с одного IP → 429 | распределённый перебор с многих IP ограничен только лимитом на username; при недоступном Valkey работает локальный (не общий) лимит | +| T6 | DoS через дорогой вход: поток Basic-запросов на несуществующих пользователей загружает CPU и RAM Argon2id | обычные запросы продолжают обслуживаться, память ограничена | не больше 4 одновременных Argon2 на реплику (иначе 503 через 3 с); dummy-хеш для неизвестных пользователей; лимиты контейнера 0.5 CPU / 512 MiB | `login_budget.py`: 32 атакующих, 40 попыток/с: CPU API в лимите, RSS 97 → ~190 MiB, обычные запросы без ошибок (p99 46 → 295 мс) | атака снижает latency; полный DDoS вне модели (нужен лимит на ingress) | +| T7 | Посторонний пользователь читает чужой чат, историю, метаданные или содержимое вложения по UUID, прикрепляет чужой файл к своему сообщению, подменяет `X-User-Id` | доступ только участникам; ответ не раскрывает существование | проверка членства на каждом запросе, единый 404; identity только из Bearer | `test_auth_acl.py`: 404 на chat/history/metadata/content/variant/post/attach; поддельный `X-User-Id` не меняет identity; Basic на бизнес-endpoint → 401 | — | +| T8 | Украденный Bearer-токен используется после logout или истечения срока | отозванная или истёкшая сессия не принимается ни одной репликой | серверные сессии (SHA-256 токена в БД), отзыв в Valkey и БД, срок в кэше не дольше `expires_at` | `test_auth_acl.py`: после logout и после истечения → 401 на обеих репликах | принудительное изменение срока в БД действует через ≤ 30 с (TTL кэша); logout действует сразу | +| T9 | Утечка пароля роли worker (дамп env или лог) | утёкший пароль быстро становится бесполезным | пароль только в OpenBao, не в env, файлах или образе; `bao.sh rotate-worker-db`; worker подхватывает новую версию KV без рестарта | `test_secret_rotation.py`: старый пароль отклонён, новые соединения используют новый, задания продолжаются | открытые до ротации соединения живут до `max_lifetime` (600 с) или принудительного завершения | +| T10 | OpenBao запечатан или упал (рестарт, атака), и сервисы стартуют с пустыми credentials или с ослабленной проверкой | секреты не подменяются ничем слабым | запущенные процессы держат полученные секреты; новый процесс ждёт до 120 с и завершается с кодом 78; распечатывает только оператор | `test_bao_sealed.py`: smoke проходит в запечатанном состоянии, перезапущенный worker ждёт и не становится healthy, после unseal восстанавливается | при длительном запечатывании токены истекают (≤ 1 ч), ротация не подхватывается; нужен оператор | +| T11 | MITM или подмена хранилища во внутренней сети: посторонний контейнер выдаёт себя за `postgres`, `minio`, `rabbitmq` или `valkey`, либо клиент откатывается на plaintext | каждый клиент проверяет сервер, plaintext не принимается | verify-full / verify=CA / ssl_check_hostname у всех клиентов; plaintext-порты закрыты (`listeners.tcp = none`, Valkey `--port 0`, `pg_hba` reject) | аудит B/C/G/I: чужой CA как trust anchor, неверный hostname → ошибка проверки; plaintext → отказ или закрытие | — | +| T12 | Секреты или содержимое утекают в telemetry: пароль, токен, текст сообщения, filename, байты изображения | telemetry не содержит секретов и пользовательского контента | захват заголовков выключен, View ограничивает атрибуты, SQL параметризован, collector `attributes/scrub` удаляет `url.query` и заголовки | `test_observability.py`: пароль, токен, маркер текста, filename, `Authorization`, signed URL, PNG (сырой и base64) в трассах и логах не найдены | технические UUID есть в span-атрибутах (обосновано в lab6) | +| T13 | Выход из контейнера или повышение привилегий через уязвимость приложения | ограниченный радиус поражения | non-root во всех 15 контейнерах, `cap_drop: ALL`, `no-new-privileges`, read-only rootfs + tmpfs, лимиты CPU/RAM/PIDs, нет Docker socket и host network | `check_hardening.py`: 15 контейнеров, 0 проблем (по `docker inspect`) | ядро хоста общее; seccomp и AppArmor по умолчанию Docker | +| T14 | Вредоносное изображение эксплуатирует уязвимость декодера (Pillow) | разбор недоверенных данных изолирован и сделан на актуальной версии | Pillow 13.0.0 (было 11.0.0 с 39 HIGH/CRITICAL CVE); декодирование в дочернем процессе с RLIMIT_AS/CPU и таймаутом; проверка magic и пикселей до декодирования | `variants_check.py` (лимиты пикселей и памяти дочернего процесса); `evidence/image_scan.txt`: CRITICAL Python 5 → 0 | новые CVE в Pillow появятся: нужен регулярный scan | ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +**Bootstrap trust:** +1. Оператор создаёт CA и leaf-сертификаты (`scripts/pki.sh init`) и bootstrap-пароли (`scripts/init-secrets.sh`) — только в `secrets/` (0700, в `.gitignore` и `.dockerignore`). +2. Оператор инициализирует OpenBao (`scripts/bao.sh init`, 3 доли, порог 2) и записывает политики, роли cert auth и секреты (`bao.sh bootstrap`). +3. Первичная identity сервиса — его TLS-ключ, выданный оператором как Docker secret. Токена, SecretID и паролей в образе, `.env` или переменных окружения нет. В OpenBao сервис входит этим сертификатом и получает токен на 15 минут с правом чтения только своего пути. + +**Renewal:** +- токен продлевается (renew-self) или запрашивается заново каждые 15 с; +- leaf-сертификаты живут 90 дней, перевыпуск — `pki.sh issue `. Файлы переписываются на месте, nginx достаточно `nginx -s reload`, API/worker — перезапуска контейнера без пересборки образа. + +**Revoke и rotation:** +- клиентский сертификат LB — `pki.sh revoke-serial`, API отклоняет его через ~1 с; +- пароль роли worker — `bao.sh rotate-worker-db`, worker подхватывает новую версию KV без рестарта; +- identity в RabbitMQ — удалить пользователя из `definitions.json`; +- в OpenBao — удалить роль `auth/cert/certs/`. + +**Компрометация identity** (пример — worker): +1. Отозвать роль worker в OpenBao, удалить пользователя `worker.client.messenger` в RabbitMQ. +2. Ротировать пароль `app_worker` и ключ S3 worker. +3. Выпустить новый сертификат worker. +4. Задания сохраняются в PostgreSQL и outbox и будут обработаны после восстановления. + +**Запечатанный OpenBao:** +- запущенные сервисы работают дальше на полученных секретах; +- перезапущенный сервис ждёт (`openbao_not_ready`) и не становится healthy; +- восстановление: `scripts/bao.sh unseal` (2 из 3 долей). + +**Владельцы:** CA, root token и доли unseal принадлежат оператору. В учебном стенде всё это у одного человека на одном хосте в `secrets/operator/` (0600). В production доли раздаются разным людям, а root token отзывается после bootstrap. + +## Scan и применимость + +Trivy 0.58.1. Образ передаётся сканеру через `docker save`, без Docker socket. Сводка: [evidence/image_scan.txt](../evidence/image_scan.txt). + +**Образ приложения: до → после.** +- Базовый образ `python:3.12.8-slim` → `3.12.16-slim`, Pillow 11.0.0 → 13.0.0, urllib3 2.3.0 → 2.8.0. +- ОС: CRITICAL 9 → 2, HIGH/CRITICAL с доступным исправлением 56 → 0. +- Python: CRITICAL 5 → 0, HIGH 43 → 3. +- Секретов в слоях не найдено. + +**Остаток:** + +| Находка | Применимость | Решение | +| --- | --- | --- | +| zlib CVE-2023-45853 (CRITICAL, исправления нет) | MiniZip (zip-архивы) не используется; Python `zlib` в `imagecheck.py` — только `inflate` для PNG | принято, следить за обновлением | +| sqlite CVE-2025-7458 (CRITICAL, исправления нет) | нужен произвольный SQL к SQLite; SQLite в приложении не используется | принято | +| Kerberos (libkrb5*), util-linux — HIGH без исправления | GSSAPI в psycopg не используется (SCRAM по TLS); util-linux — инструменты образа, не код приложения | принято | +| starlette CVE-2025-62727 (Range в FileResponse) | `FileResponse` и `StaticFiles` не используются | неприменимо | +| starlette CVE-2026-48818 (StaticFiles на Windows) | Linux, `StaticFiles` нет | неприменимо | +| starlette CVE-2026-54283 (лимиты `request.form()`) | `form()` и multipart не используются: JSON и сырой upload с собственными лимитами 64 KiB / 10 MiB | неприменимо | +| starlette CVE-2026-48710 / 54282 (`request.url` собирается из Host и пути) | частично: guard использовал `request.url.path` для освобождения health-путей от проверок готовности | **исправлено**: везде `request.scope["path"]` (сырой путь маршрутизации) | +| nginx 1.27.3 (CRITICAL 3, исправления есть) | ingress | **обновлено** до 1.30.6-alpine | +| OpenBao 2.1.0 (Go stdlib CVE в TLS-сервере, исправления есть) | хранилище секретов | **обновлено** до 2.6.5; хранилище `file` подхвачено без миграции | + +Starlette не обновлялась: для этого нужен мажорный переход FastAPI 0.115 → 0.16x. Это следующий шаг, но не в рамках лабораторной, потому что применимых уязвимостей не осталось. diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..ea05592 --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Every Prometheus query of the provisioned dashboard must parse and return data (after some traffic). + +python3 student-tests/check_dashboard.py +""" +import json +import sys +import urllib.error +from pathlib import Path + +import obs + +ROOT = Path(__file__).resolve().parents[1] +dash = json.loads((ROOT / "infra/grafana/dashboards/messenger.json").read_text(encoding="utf-8")) +bad = 0 +for panel in dash["panels"]: + for t in panel.get("targets", []): + if t.get("datasource", {}).get("type") != "prometheus": + continue + expr = t["expr"].replace("$__rate_interval", "2m") + try: + rows = obs.prom(expr) + status = "ok %3d series" % len(rows) if rows else "EMPTY" + except urllib.error.HTTPError as exc: + status, rows = "ERROR %s" % exc.read()[:120], None + if not rows: + bad += 1 + print("%-16s %-48s %s" % (status, panel["title"][:48], t.get("legendFormat", ""))) +print("panels: %d, prometheus queries without data: %d" % (len(dash["panels"]), bad)) + + +# ---------------------------------------------------------------- the same panels executed BY GRAFANA (/api/ds/query) +# Proves the provisioned datasources work from Grafana itself (Prometheus, Loki and Tempo), with Grafana macros. +import base64 # noqa: E402 +import urllib.request # noqa: E402 + +password = (ROOT / "secrets/bootstrap/grafana_admin_password").read_text().strip() +import ssl # noqa: E402 +GRAFANA_TLS = ssl.create_default_context(cafile=str(ROOT / "secrets/pki/ca.crt")) # lab7: Grafana is HTTPS-only +auth = "Basic " + base64.b64encode(("admin:" + password).encode()).decode() +via_bad = 0 +for panel in dash["panels"]: + for t in panel.get("targets", []): + q = dict(t) + q.setdefault("datasource", panel.get("datasource")) + if q["datasource"]["type"] == "tempo": + # Grafana 11 runs TraceQL search in the browser through the datasource proxy, not /api/ds/query + # ("backend TraceQL search queries are not supported"): check exactly that proxy path. + import time as _t + import urllib.parse as _p + url = ("https://localhost:3000/api/datasources/proxy/uid/tempo/api/search?" + + _p.urlencode({"q": q["query"], "limit": q.get("limit", 20), "start": int(_t.time()) - 1800, + "end": int(_t.time())})) + req = urllib.request.Request(url, headers={"Authorization": auth}) + try: + with urllib.request.urlopen(req, timeout=60, context=GRAFANA_TLS) as response: + n = len(json.loads(response.read()).get("traces", [])) + status = "ok %5d traces" % n if n else "EMPTY" + except urllib.error.HTTPError as exc: + status = "ERROR HTTP %s" % exc.code + if not status.startswith("ok"): + via_bad += 1 + print("grafana %-8s %-18s %-48s (datasource proxy)" % ("tempo", status, panel["title"][:48])) + continue + body = json.dumps({"queries": [q], "from": "now-30m", "to": "now"}).encode() + req = urllib.request.Request("https://localhost:3000/api/ds/query", data=body, method="POST", + headers={"Content-Type": "application/json", "Authorization": auth}) + try: + with urllib.request.urlopen(req, timeout=60, context=GRAFANA_TLS) as response: + res = json.loads(response.read())["results"][q["refId"]] + frames = res.get("frames", []) + rows = sum(len((f.get("data") or {}).get("values", [[]])[0] or []) for f in frames) + status = "ERROR %s" % res["error"][:100] if res.get("error") else ("ok %5d values" % rows if rows else "EMPTY") + except urllib.error.HTTPError as exc: + status = "ERROR HTTP %s %s" % (exc.code, exc.read()[:100]) + if not status.startswith("ok"): + via_bad += 1 + print("grafana %-8s %-18s %-48s" % (q["datasource"]["type"], status, panel["title"][:48])) +print("via Grafana: queries without data or with errors: %d" % via_bad) diff --git a/student-tests/check_hardening.py b/student-tests/check_hardening.py new file mode 100644 index 0000000..c8ba0a0 --- /dev/null +++ b/student-tests/check_hardening.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Level 3-4 hardening audit from the REAL container state (docker inspect), not from compose.yaml. + +Checks every running container of the project: non-root user, not privileged, no host network/PID/IPC, no Docker +socket mount, all capabilities dropped (no cap_add), no-new-privileges, read-only root FS (+ tmpfs list), CPU/memory/ +PIDs limits, and that only the allowed ports are published, and only on 127.0.0.1. +python3 student-tests/check_hardening.py [--output evidence/hardening.json] +""" +import argparse +import json +import subprocess +import sys + +ALLOWED_PUBLISHED = {"8443": "nginx business ingress (TLS)", "3000": "Grafana (HTTPS + login)", + "9090": "Prometheus (mTLS operator)", "3100": "Loki (mTLS operator)", "3200": "Tempo (mTLS operator)"} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--output", default="") + a = p.parse_args() + ids = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab7"], + capture_output=True, text=True).stdout.split() + rows, failures = [], [] + for info in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout): + name = info["Name"].strip("/").replace("mtusi-lab7-", "") + hc, cfg = info["HostConfig"], info["Config"] + user = cfg.get("User") or "root(image default)" + mounts = [m.get("Source", "") for m in info["Mounts"]] + published = [] + for port, binds in (info["NetworkSettings"]["Ports"] or {}).items(): + for b in binds or []: + published.append("%s:%s->%s" % (b["HostIp"], b["HostPort"], port)) + row = { + "container": name, "user": user, "privileged": hc["Privileged"], "network": hc["NetworkMode"], + "pid": hc.get("PidMode") or "private", "cap_drop": hc.get("CapDrop"), "cap_add": hc.get("CapAdd"), + "no_new_privileges": "no-new-privileges:true" in (hc.get("SecurityOpt") or []), + "read_only_rootfs": hc["ReadonlyRootfs"], "tmpfs": sorted((hc.get("Tmpfs") or {}).keys()), + "cpus": hc["NanoCpus"] / 1e9, "memory_mib": hc["Memory"] // 1048576, "pids_limit": hc.get("PidsLimit"), + "docker_socket": any("docker.sock" in m for m in mounts), "published": published, + } + problems = [] + if user.split(":")[0] in ("", "0", "root", "root(image default)"): + problems.append("runs as root") + if row["privileged"] or row["network"] == "host" or row["pid"] == "host" or row["docker_socket"]: + problems.append("privileged/host namespace/docker socket") + if row["cap_drop"] != ["ALL"] or row["cap_add"]: + problems.append("capabilities not fully dropped") + if not row["no_new_privileges"]: + problems.append("no-new-privileges missing") + if not row["read_only_rootfs"]: + problems.append("writable root FS") + if not (row["cpus"] and row["memory_mib"] and row["pids_limit"]): + problems.append("missing CPU/memory/PIDs limit") + for pub in published: + host_ip, rest = pub.split(":", 1) + if host_ip != "127.0.0.1" or rest.split("->")[0] not in ALLOWED_PUBLISHED: + problems.append("unexpected published port " + pub) + row["problems"] = problems + rows.append(row) + failures += ["%s: %s" % (name, x) for x in problems] + rows.sort(key=lambda r: r["container"]) + print("%-16s %-11s %-6s %-5s %-4s %-5s %-9s %-24s %s" % ("container", "user", "capDrop", "nnp", "ro", "cpus", + "mem/pids", "published", "problems")) + for r in rows: + print("%-16s %-11s %-6s %-5s %-4s %-5s %-9s %-24s %s" % ( + r["container"][:16], r["user"][:11], "ALL" if r["cap_drop"] == ["ALL"] else r["cap_drop"], + r["no_new_privileges"], r["read_only_rootfs"], r["cpus"], "%d/%s" % (r["memory_mib"], r["pids_limit"]), + ",".join(x.split("->")[0] for x in r["published"]) or "-", "; ".join(r["problems"]) or "ok")) + print("published ports allowed: %s" % ALLOWED_PUBLISHED) + print("%d containers, %d problems" % (len(rows), len(failures))) + if a.output: + with open(a.output, "w", encoding="utf-8") as handle: + json.dump(rows, handle, ensure_ascii=False, indent=1) + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main() diff --git a/student-tests/login_budget.py b/student-tests/login_budget.py new file mode 100644 index 0000000..3927246 --- /dev/null +++ b/student-tests/login_budget.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Argon2id parameters and the login DoS budget, measured on the stand (level 3 + level 4). + + bench inside api-1 (same CPU limit 0.5 core as production replicas): time and peak RSS of one hash for + several parameter sets -> justification of m=19 MiB, t=2, p=1 + flood N concurrent attackers send wrong-password logins (distinct usernames, so per-user limits do not + help; the per-IP limit must be raised for this run to model a distributed attacker) while a probe + client sends normal requests; reports login throughput, 503 hash_busy share, API CPU/RSS and probe + latency vs. a baseline without the flood. + + LOGIN_RATE_LIMIT_PER_IP=1000000 docker compose --env-file .env up -d --no-deps api-1 api-2 # experiment only + python3 student-tests/login_budget.py --output evidence/login_budget.json + docker compose --env-file .env up -d --no-deps api-1 api-2 # restore limits +""" +import argparse +import base64 +import http.client +import json +import ssl +import statistics +import subprocess +import threading +import time +import uuid +from pathlib import Path + +from stand import ROOT, Client, log + +BENCH = r''' +import json, resource, time +from argon2 import PasswordHasher +out = [] +for m, t, p in ((19456, 2, 1), (12288, 3, 1), (47104, 1, 1), (65536, 3, 1), (65536, 3, 4)): + h = PasswordHasher(time_cost=t, memory_cost=m, parallelism=p) + h.hash("warm-up-password") + times = [] + for _ in range(10): + t0 = time.perf_counter(); h.hash("benchmark-password-123"); times.append((time.perf_counter() - t0) * 1000) + times.sort() + out.append({"m_kib": m, "t": t, "p": p, "p50_ms": round(times[5], 1), "max_ms": round(times[-1], 1), + "peak_rss_mib": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1)}) +print(json.dumps(out)) +''' + + +def ctx(): + return ssl.create_default_context(cafile=str(ROOT / "secrets/pki/ca.crt")) + + +def request(conn, method, path, headers, body=None): + conn.request(method, path, body=body, headers=headers) + r = conn.getresponse() + r.read() + return r.status + + +def cgroup_cpu(): + """CPU seconds of api-1 + api-2 from the Docker API (exact over a window).""" + host = subprocess.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + capture_output=True, text=True).stdout.strip().replace("unix://", "") + total = 0.0 + for name in ("mtusi-lab7-api-1-1", "mtusi-lab7-api-2-1"): + raw = subprocess.run(["curl", "-s", "--unix-socket", host, + "http://localhost/containers/%s/stats?stream=false&one-shot=true" % name], + capture_output=True, text=True).stdout + total += json.loads(raw)["cpu_stats"]["cpu_usage"]["total_usage"] / 1e9 + return total + + +def max_rss(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, text=True).stdout + return {l.split()[0]: l.split()[1] for l in out.splitlines() if "api-" in l and "lab7" in l} + + +def probe(actor, chat, stop, lat): + conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30) + headers = {"Authorization": "Bearer " + actor["token"], "Content-Type": "application/json"} + while not stop.is_set(): + t0 = time.perf_counter() + try: + h = dict(headers, **{"Idempotency-Key": uuid.uuid4().hex}) + status = request(conn, "POST", "/api/v1/chats/%s/messages" % chat["id"], h, b'{"text":"probe"}') + lat.append((time.perf_counter() - t0, status)) + except (OSError, http.client.HTTPException): + conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30) + time.sleep(0.2) + + +def attacker(stop, results): + conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30) + while not stop.is_set(): + name = "nouser_" + uuid.uuid4().hex[:12] + auth = base64.b64encode((name + ":wrong-password-123").encode()).decode() + t0 = time.perf_counter() + try: + status = request(conn, "POST", "/api/v1/auth/sessions", {"Authorization": "Basic " + auth}) + except (OSError, http.client.HTTPException): + conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30) + status = "conn_error" + results.append((time.perf_counter() - t0, status)) + + +def phase(actor, chat, attackers, duration): + stop, lat, logins = threading.Event(), [], [] + threads = [threading.Thread(target=probe, args=(actor, chat, stop, lat))] + threads += [threading.Thread(target=attacker, args=(stop, logins)) for _ in range(attackers)] + cpu0, t0 = cgroup_cpu(), time.time() + for t in threads: + t.start() + rss = [] + while time.time() - t0 < duration: + time.sleep(5) + rss.append(max_rss()) + stop.set() + for t in threads: + t.join(40) + elapsed = time.time() - t0 + cpu = (cgroup_cpu() - cpu0) / elapsed * 100 + ok = sorted(x for x, s in lat if s == 201) + codes = {} + for _, s in logins: + codes[str(s)] = codes.get(str(s), 0) + 1 + login_lat = sorted(x for x, s in logins if s == 401) + pct = lambda v, q: round(v[min(len(v) - 1, int(q * len(v)))] * 1000, 1) if v else None # noqa: E731 + return {"attackers": attackers, "seconds": round(elapsed), "login_attempts_per_s": round(len(logins) / elapsed, 1), + "login_status_counts": codes, "login_401_p50_ms": pct(login_lat, .5), "login_401_p99_ms": pct(login_lat, .99), + "probe_ok": len(ok), "probe_errors": sum(1 for _, s in lat if s != 201), + "probe_p50_ms": pct(ok, .5), "probe_p99_ms": pct(ok, .99), + "api_cpu_pct_of_one_core_both_replicas": round(cpu, 1), "api_rss_samples": rss[-1] if rss else {}} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--duration", type=int, default=30) + p.add_argument("--attackers", type=int, default=32) + p.add_argument("--output", default=str(ROOT / "evidence/login_budget.json")) + a = p.parse_args() + bench = json.loads(subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", BENCH], cwd=ROOT, + capture_output=True, text=True, check=True).stdout) + for row in bench: + log("argon2id m=%(m_kib)d KiB t=%(t)d p=%(p)d: p50 %(p50_ms)s ms, max %(max_ms)s ms, peak RSS %(peak_rss_mib)s MiB" % row) + c = Client() + alice, _bob, _eve, chat = c.bootstrap() + base = phase(alice, chat, 0, a.duration) + log("baseline: %s" % base) + flood = phase(alice, chat, a.attackers, a.duration) + log("flood: %s" % flood) + result = {"argon2_bench_api_container_0_5_cpu": bench, "baseline": base, "flood": flood, + "config": {"PASSWORD_HASH_CONCURRENCY_per_replica": 4, "PASSWORD_HASH_QUEUE_SECONDS": 3, + "replicas": 2, "api_cpu_limit": 0.5, "api_mem_limit_mib": 512}} + Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(result, ensure_ascii=False, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/mtls_probe.py b/student-tests/mtls_probe.py new file mode 100644 index 0000000..d56f886 --- /dev/null +++ b/student-tests/mtls_probe.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""In-network probe: GET /health/ready on an API replica with a given client certificate from /pki. +docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./student-tests:/audit:ro --entrypoint python \ + worker /audit/mtls_probe.py api-1 lb-client [lb-client-old ...] -> one JSON line per certificate +""" +import json +import sys + +sys.path.insert(0, "/audit") +import tls_audit # noqa: E402 + +host = sys.argv[1] +for name in sys.argv[2:]: + verdict, detail = tls_audit.http_get(host, 8080, "/health/ready", name) + print(json.dumps({"host": host, "cert": name, "verdict": verdict, "detail": detail})) diff --git a/student-tests/obs.py b/student-tests/obs.py new file mode 100644 index 0000000..648b672 --- /dev/null +++ b/student-tests/obs.py @@ -0,0 +1,146 @@ +"""Query helpers for the telemetry backends (stdlib only). Backends listen on 127.0.0.1 (compose.yaml). + +Never prints tokens or passwords; callers print only ids, counts, span names and attributes. +""" +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request + +import ssl +from pathlib import Path + +# lab7: the backends accept only clients of the "Telemetry Clients CA": evidence scripts use the operator cert. +PROM = os.environ.get("PROM_URL", "https://localhost:9090") +LOKI = os.environ.get("LOKI_URL", "https://localhost:3100") +TEMPO = os.environ.get("TEMPO_URL", "https://localhost:3200") +_PKI = Path(__file__).resolve().parents[1] / "secrets/pki" +TLS = ssl.create_default_context(cafile=str(_PKI / "ca.crt")) +TLS.load_cert_chain(str(_PKI / "operator.crt"), str(_PKI / "operator.key")) + + +def _get(url, params=None, timeout=15): + if params: + url += "?" + urllib.parse.urlencode(params) + with urllib.request.urlopen(url, timeout=timeout, context=TLS) as response: + return json.loads(response.read()) + + +def prom(expr, at=None): + """Instant query -> list of (labels, float value).""" + params = {"query": expr} + if at: + params["time"] = at + data = _get(PROM + "/api/v1/query", params)["data"]["result"] + return [(r["metric"], float(r["value"][1])) for r in data] + + +def prom_scalar(expr, default=0.0, at=None): + rows = prom(expr, at) + return rows[0][1] if rows else default + + +def prom_range(expr, start, end, step=5): + data = _get(PROM + "/api/v1/query_range", {"query": expr, "start": start, "end": end, "step": step}) + return data["data"]["result"] + + +def alerts(): + """{alertname: state} for pending/firing alerts in Prometheus.""" + out = {} + for a in _get(PROM + "/api/v1/alerts")["data"]["alerts"]: + out[a["labels"]["alertname"]] = a["state"] + return out + + +def loki(query, minutes=15, limit=500): + end = time.time_ns() + data = _get(LOKI + "/loki/api/v1/query_range", {"query": query, "limit": limit, "direction": "forward", + "start": end - minutes * 60 * 10**9, "end": end}) + lines = [] + for stream in data["data"]["result"]: + for ts, line in stream["values"]: + lines.append({"ts": int(ts), "line": line, **stream["stream"]}) + return sorted(lines, key=lambda item: item["ts"]) + + +def tempo_search(traceql, minutes=30, limit=100, start=None, end=None): + end = int(end or time.time() + 60) + start = int(start or end - 60 - minutes * 60) + data = _get(TEMPO + "/api/search", {"q": traceql, "start": start, "end": end, "limit": limit, "spss": 50}, + timeout=60) + return data.get("traces", []) + + +def _hex(value): + return base64.b64decode(value).hex() if value else None + + +def _attr(value): + for key in ("stringValue", "intValue", "boolValue", "doubleValue"): + if key in value: + return value[key] + return None + + +def tempo_trace(trace_id): + """Flat list of spans: service, name, kind, span_id, parent_id, duration_ms, status, attrs (dict).""" + try: + data = _get(TEMPO + "/api/traces/" + trace_id) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return [] + raise + spans = [] + for batch in data.get("batches", []): + service = {a["key"]: _attr(a["value"]) for a in batch["resource"]["attributes"]}.get("service.name") + for scope in batch.get("scopeSpans", []): + for s in scope["spans"]: + spans.append({ + "service": service, "name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""), + "span_id": _hex(s["spanId"]), "parent_id": _hex(s.get("parentSpanId")), + "start": int(s["startTimeUnixNano"]), + "duration_ms": (int(s["endTimeUnixNano"]) - int(s["startTimeUnixNano"])) / 1e6, + "status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", "") or "UNSET", + "attrs": {a["key"]: _attr(a["value"]) for a in s.get("attributes", [])}, + }) + return sorted(spans, key=lambda item: item["start"]) + + +def tempo_raw(trace_id): + with urllib.request.urlopen(TEMPO + "/api/traces/" + trace_id, timeout=15, context=TLS) as response: + return response.read().decode("utf-8", "replace") + + +def print_tree(spans, out=print, skip_db=False): + children = {} + ids = {s["span_id"] for s in spans} + for s in spans: + children.setdefault(s["parent_id"] if s["parent_id"] in ids else None, []).append(s) + + def walk(parent, depth): + for s in children.get(parent, []): + if skip_db and s["kind"] == "CLIENT" and s["name"].split()[0] in ("SELECT", "UPDATE", "INSERT", "DELETE"): + continue + extra = {k: v for k, v in s["attrs"].items() if k.startswith(("messenger.", "messaging.message", "db.system", + "http.response.status_code", "rpc.method"))} + out("%s%-8s %-9s %-44s %8.1f ms %-5s %s" % (" " * depth, s["service"].replace("messenger-", ""), s["kind"], + s["name"][:44], s["duration_ms"], s["status"], + json.dumps(extra, ensure_ascii=False))) + walk(s["span_id"], depth + 1) + + walk(None, 0) + + +def wait_for(predicate, seconds=60, interval=2): + deadline = time.monotonic() + seconds + value = None + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(interval) + return value diff --git a/student-tests/openloop.py b/student-tests/openloop.py new file mode 100644 index 0000000..68a1514 --- /dev/null +++ b/student-tests/openloop.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Open-loop load generator (stdlib only): fixed arrival rate, latency measured from the SCHEDULED send time, +so a slow server cannot slow down the generator (no coordinated omission while threads are available). + +Mix per arrival: 75% POST message, 25% GET history (limit=20); plus an independent upload stream (sample.png). +Rare injected requests (FAULT_INJECTION=1 on the stand): `X-Fault: slow` (+600 ms) and `X-Fault: error` (500) +are added to GET history with the given percentages, to measure whether such traces survive sampling. + + python3 student-tests/openloop.py --rate 40 --duration 60 --upload-rate 1 --slow-pct 1 --error-pct 1 \ + --output evidence/x.json +Tokens stay in memory; the summary contains counts, percentiles and timestamps only. +""" +import argparse +import http.client +import json +import os +import queue +import random +import ssl +import sys +import threading +import time +import uuid +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +os.environ.setdefault("CA_FILE", str(ROOT / "secrets/pki/ca.crt")) +from client import Client # noqa: E402 + +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def percentile(values, p): + if not values: + return None + values = sorted(values) + k = max(0, min(len(values) - 1, int(round(p / 100.0 * len(values) + 0.5)) - 1)) + return values[k] + + +class Load: + def __init__(self, base_url, users=4, threads=64, timeout=30.0): + self.base = urlsplit(base_url) + self.tls = ssl.create_default_context(cafile=os.environ.get("CA_FILE") or str(ROOT / "secrets/pki/ca.crt")) + self.threads = threads + self.timeout = timeout + c = Client(base_url) + self.actors = [] + for i in range(users): + a, b = c.user("load%da" % i), c.user("load%db" % i) + chat = c.call("POST", "/api/v1/chats", 201, actor=a, + body={"title": "load %d" % i, "member_ids": [a["id"], b["id"]]}, schema="Chat")["json"] + self.actors.append((a["token"], chat["id"])) + self.lock = threading.Lock() + self.results = [] + self.local = threading.local() + + def conn(self): + c = getattr(self.local, "conn", None) + if c is None: + if self.base.scheme == "https": # lab7: TLS verified against the stand CA, never unverified + c = http.client.HTTPSConnection(self.base.hostname, self.base.port or 443, timeout=self.timeout, + context=self.tls) + else: + c = http.client.HTTPConnection(self.base.hostname, self.base.port or 80, timeout=self.timeout) + self.local.conn = c + return c + + def request(self, method, path, body=None, headers=None): + for attempt in (1, 2): # second attempt only for a dropped keep-alive connection BEFORE sending + conn = self.conn() + try: + conn.request(method, path, body=body, headers=headers or {}) + response = conn.getresponse() + response.read() + return response.status, response.getheader("X-Request-Id") + except (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError) as exc: + conn.close() + self.local.conn = None + if attempt == 2: + return "conn_error:" + type(exc).__name__, None + except (OSError, http.client.HTTPException) as exc: + conn.close() + self.local.conn = None + return "error:" + type(exc).__name__, None + return "error", None + + def one(self, kind, fault, scheduled): + token, chat = random.choice(self.actors) + headers = {"Authorization": "Bearer " + token} + if fault: + headers["X-Fault"] = fault + if kind == "post_message": + headers.update({"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}) + status, rid = self.request("POST", "/api/v1/chats/%s/messages" % chat, + json.dumps({"text": "нагрузка " + uuid.uuid4().hex[:8]}).encode(), headers) + elif kind == "get_history": + status, rid = self.request("GET", "/api/v1/chats/%s/messages?limit=20" % chat, None, headers) + else: + headers["Content-Type"] = "image/png" + status, rid = self.request("POST", "/api/v1/chats/%s/attachments?filename=load.png" % chat, SAMPLE, headers) + done = time.monotonic() + with self.lock: + self.results.append({"kind": kind, "fault": fault, "status": status, "latency": done - scheduled, + "at": time.time(), "rid": rid}) + + def run(self, rate, duration, upload_rate=0.0, slow_pct=0.0, error_pct=0.0, stop_event=None): + work = queue.Queue() + start = time.monotonic() + self.started_at = time.time() + + def worker(): + while True: + item = work.get() + if item is None: + return + self.one(*item) + + pool = [threading.Thread(target=worker, daemon=True) for _ in range(self.threads)] + for t in pool: + t.start() + + def schedule(r, picker): + if r <= 0: + return + n = 0 + while True: + due = start + n / r + if due - start >= duration or (stop_event and stop_event.is_set()): + return + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + work.put(picker() + (due,)) + n += 1 + + def pick_main(): + if random.random() < 0.75: + return ("post_message", None) + roll = random.random() * 100 + fault = "slow" if roll < slow_pct * 4 else "error" if roll < (slow_pct + error_pct) * 4 else None + return ("get_history", fault) # 25% share * 4 = requested share of ALL requests + + schedulers = [threading.Thread(target=schedule, args=(rate, pick_main), daemon=True), + threading.Thread(target=schedule, args=(upload_rate, lambda: ("upload", None)), daemon=True)] + for t in schedulers: + t.start() + for t in schedulers: + t.join() + backlog = work.qsize() + for _ in pool: + work.put(None) + for t in pool: + t.join(self.timeout + 5) + self.finished_at = time.time() + return self.summary(rate, duration, upload_rate, slow_pct, error_pct, backlog) + + def summary(self, rate, duration, upload_rate, slow_pct, error_pct, backlog): + res = self.results + ok = [r for r in res if isinstance(r["status"], int) and r["status"] < 400] + plain_ok = [r["latency"] for r in ok if r["fault"] is None and r["kind"] != "upload"] + out = { + "profile": {"rate_rps": rate, "duration_s": duration, "upload_rate_rps": upload_rate, + "slow_pct": slow_pct, "error_pct": error_pct, "threads": self.threads, + "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, + "started_at": self.started_at, "finished_at": self.finished_at, "generator_backlog_at_end": backlog, + "requests": len(res), "ok": len(ok), + "successful_rps": round(len(ok) / max(duration, 1e-9), 2), + "status_counts": {}, + "server_errors_unexpected": sum(1 for r in res if r["fault"] != "error" and ( + not isinstance(r["status"], int) or r["status"] >= 500)), + "latency_ms_plain_ok": {p: round(percentile(plain_ok, q) * 1000, 2) if plain_ok else None + for p, q in (("p50", 50), ("p95", 95), ("p99", 99), ("max", 100))}, + "faults_sent": {"slow": sum(1 for r in res if r["fault"] == "slow"), + "error": sum(1 for r in res if r["fault"] == "error")}, + "by_kind": {}, + } + for r in res: + key = str(r["status"]) + out["status_counts"][key] = out["status_counts"].get(key, 0) + 1 + for kind in ("post_message", "get_history", "upload"): + lat = [r["latency"] for r in ok if r["kind"] == kind and r["fault"] is None] + out["by_kind"][kind] = {"n": sum(1 for r in res if r["kind"] == kind), + "p50_ms": round(percentile(lat, 50) * 1000, 2) if lat else None, + "p99_ms": round(percentile(lat, 99) * 1000, 2) if lat else None} + return out + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--base-url", default="https://localhost:8443") + p.add_argument("--rate", type=float, default=40) + p.add_argument("--duration", type=float, default=60) + p.add_argument("--upload-rate", type=float, default=0) + p.add_argument("--slow-pct", type=float, default=0) + p.add_argument("--error-pct", type=float, default=0) + p.add_argument("--threads", type=int, default=64) + p.add_argument("--output") + a = p.parse_args() + load = Load(a.base_url, threads=a.threads) + summary = load.run(a.rate, a.duration, a.upload_rate, a.slow_pct, a.error_pct) + text = json.dumps(summary, ensure_ascii=False, indent=2) + print(text) + if a.output: + Path(a.output).write_text(text + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/operator_checks.sh b/student-tests/operator_checks.sh new file mode 100755 index 0000000..1d4f220 --- /dev/null +++ b/student-tests/operator_checks.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Checks that need the OPERATOR material of OpenBao (unseal shares / root token in secrets/operator/). +# Run by the operator from the repository root: sh student-tests/operator_checks.sh +# Every step writes its output to evidence/; nothing secret is printed. +set -u +cd "$(dirname "$0")/.." +CA="$PWD/secrets/pki/ca.crt" +run() { name=$1; shift; printf '%-28s ' "$name"; if "$@" > "evidence/$name.txt" 2>&1; then echo PASS; else echo "FAIL (see evidence/$name.txt)"; fi; } +ps_state() { docker compose ps --format '{{.Name}} {{.Status}}' | grep -vE 'migrate|minio-init'; } + +echo "1/5 unseal OpenBao and wait for the services" +sh scripts/bao.sh unseal +docker compose --env-file .env up -d --wait --wait-timeout 240 >/dev/null 2>&1 && echo " stand healthy" + +echo "2/5 operator checks" +run secret_rotation caffeinate -i python3 -u student-tests/test_secret_rotation.py +run bao_sealed caffeinate -i python3 -u student-tests/test_bao_sealed.py + +echo "3/5 persistence (restart of every store incl. OpenBao + unseal, volumes kept)" +{ echo "# $(date -u +%FT%TZ) scripts/restart.sh"; echo '$ docker compose ps (before)'; ps_state + echo '$ make test-persistence ACTION_SCRIPT=scripts/restart.sh' + caffeinate -i make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE="$CA" 2>&1 | tail -2 + echo '$ docker compose ps (after)'; ps_state; } > evidence/persistence.txt 2>&1 +grep -q "^PASS" evidence/persistence.txt && echo " persistence PASS" || echo " persistence FAIL (evidence/persistence.txt)" + +echo "4/5 failover of each API replica" +sleep 20 +{ echo "# $(date -u +%FT%TZ)" + for i in 1 2; do + echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run $i)" + caffeinate -i make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE="$CA" 2>&1 | tail -3 + docker compose ps --format '{{.Name}} {{.Status}}' | grep api + echo '$ docker compose start api-1 api-2'; docker compose start api-1 api-2 >/dev/null 2>&1; sleep 30 + done; } > evidence/failover.txt 2>&1 +echo " failover: $(grep -c '^PASS' evidence/failover.txt)/2 PASS" + +echo "5/5 smoke and full TLS audit after everything" +run smoke make test CA_FILE="$CA" +run tls_audit_final docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro \ + -v ./student-tests:/audit:ro -v ./evidence:/evidence --entrypoint python worker /audit/tls_audit.py \ + --output /evidence/tls_audit.json +tail -1 evidence/tls_audit_final.txt diff --git a/student-tests/queue_load.py b/student-tests/queue_load.py new file mode 100644 index 0000000..65f03b3 --- /dev/null +++ b/student-tests/queue_load.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB. + +Each run: + 1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH + 2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses) + 3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second + 4. waits until every accepted upload is ready/failed and computes from attachment_jobs: + queue lag = first_started_at - created_at (time a job waited for a worker) + time to ready = finished_at - attachment.created_at + throughput = finished jobs / (last finish - first arrival) + Rejections (503 queue_full from admission control) are counted, not retried. + + python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2 + python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30 +Results are appended as JSON lines to evidence/track1/results.jsonl. +""" +import argparse +import json +import os +import statistics +import subprocess +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready + +OUT = ROOT / "evidence/track1" +SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")} + +GEN = r''' +import io, random, sys +from PIL import Image, ImageChops, ImageDraw +w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3] +if fmt == "JPEG": # photo-like noise: realistic JPEG sizes + chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)] + grain = Image.effect_noise((w, h), 12) + img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan]) +else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling) + img = Image.linear_gradient("L").resize((w, h)).convert("RGB") + draw, rnd = ImageDraw.Draw(img), random.Random(1) + for _ in range(400): + x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300) + draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256))) +buf = io.BytesIO() +img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6})) +sys.stdout.buffer.write(buf.getvalue()) +''' + + +def image(size): + w, h, fmt = SIZES[size] + cache = OUT / ("fixture-%s.%s" % (size, fmt.lower())) + if not cache.exists(): + OUT.mkdir(parents=True, exist_ok=True) + data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt], + cwd=ROOT, capture_output=True, check=True).stdout + cache.write_bytes(data) + return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg" + + +def set_worker(replicas, concurrency, prefetch, cpus="2.00"): + env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency), + WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus) + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"], + cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) + + +def set_admission(max_pending): + env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending)) + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"], + cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) + + +def parse_mem(text): + value = text.split("/")[0].strip() + for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)): + if value.endswith(unit): + return float(value[: -len(unit)]) * mult + return 0.0 + + +class Sampler(threading.Thread): + def __init__(self, tag): + super().__init__(daemon=True) + self.tag = tag + self.stop = threading.Event() + self.rss, self.pending, self.cpu = [], [], [] + + def run(self): + while not self.stop.is_set(): + ids = compose("ps", "-q", "worker", check_rc=False).split() + if ids: + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids], + capture_output=True, text=True).stdout + rows = [line.split("|") for line in out.splitlines() if "|" in line] + self.rss.append(sum(parse_mem(r[0]) for r in rows)) + self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows)) + try: + self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" + % self.tag)[0][0])) + except Exception: + pass + self.stop.wait(1.0) + + +def pct(values, q): + if not values: + return None + values = sorted(values) + return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3) + + +def drive(c, actor, chat, body, ctype, rate, duration, tag): + """Open-loop: request i is sent at t0 + i/rate regardless of earlier responses.""" + total = int(rate * duration) + results = {"accepted": 0, "rejected_503": 0, "other_errors": 0} + lock = threading.Lock() + + def one(i): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i))) + try: + r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype}) + key = "accepted" if r["status"] == 202 else "rejected_503" + except Exception: # connection errors, timeouts: counted, never retried + key = "other_errors" + with lock: + results[key] += 1 + + t0 = time.monotonic() + with ThreadPoolExecutor(max_workers=64) as pool: + for i in range(total): + delay = t0 + i / rate - time.monotonic() + if delay > 0: + time.sleep(delay) + pool.submit(one, i) + results["send_seconds"] = round(time.monotonic() - t0, 2) + results["offered"] = total + return results + + +def wait_drained(tag, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0]) + if left == 0: + return True + time.sleep(1) + return False + + +def job_metrics(tag): + rows = sql( + "SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at)," + " j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%'" % tag) + done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"] + lag = [s - a for a, s, f in done] + ttr = [f - a for a, s, f in done] + span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0 + return { + "succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"), + "retries": sum(int(r[4]) - 1 for r in rows), + "throughput_jobs_s": round(len(done) / span, 3) if span else None, + "lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0), + "ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0), + } + + +def record(summary): + OUT.mkdir(parents=True, exist_ok=True) + with open(OUT / "results.jsonl", "a") as handle: + handle.write(json.dumps(summary, ensure_ascii=False) + "\n") + log(json.dumps(summary, ensure_ascii=False)) + + +def experiment(args, label, max_pending=None): + c = Client(timeout=60) + wait_http_ready(c) + alice = c.user("load") + chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]}, + schema="Chat")["json"] + body, ctype = image(args.size) + set_worker(args.replicas, args.concurrency, args.prefetch) + time.sleep(2) + tag = uuid.uuid4().hex[:8] + sampler = Sampler(tag) + sampler.start() + log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % ( + label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch)) + sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag) + load_end = time.monotonic() + drained = wait_drained(tag, args.drain_timeout) + drain_seconds = round(time.monotonic() - load_end, 2) + sampler.stop.set() + sampler.join() + summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate, + "duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency, + "prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained, + "drain_after_load_s": drain_seconds, **job_metrics(tag), + "pending_max": max(sampler.pending or [0]), + "worker_mem_peak_mib": round(max(sampler.rss or [0]), 1), + "worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None, + "worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1), + "at": time.strftime("%Y-%m-%dT%H:%M:%S")} + return summary, c, alice, chat + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=["run", "overload"]) + parser.add_argument("--size", choices=sorted(SIZES), default="medium") + parser.add_argument("--rate", type=float, default=4) + parser.add_argument("--duration", type=float, default=30) + parser.add_argument("--replicas", type=int, default=1) + parser.add_argument("--concurrency", type=int, default=2) + parser.add_argument("--prefetch", type=int, default=2) + parser.add_argument("--max-pending", type=int, default=30) + parser.add_argument("--drain-timeout", type=float, default=600) + parser.add_argument("--label", default="") + args = parser.parse_args() + if args.mode == "run": + summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch)) + record(summary) + return + # Overload: admission limit on, offered rate far above capacity, then recovery. + set_admission(args.max_pending) + try: + summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending) + # Recovery probe: after the backlog drained, a fresh upload must be ready quickly again. + body, ctype = image("small") + t = time.monotonic() + att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body, + headers={"Content-Type": ctype}, schema="Attachment")["json"] + while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready": + time.sleep(0.2) + summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2) + record(summary) + finally: + set_admission(200) + + +if __name__ == "__main__": + main() diff --git a/student-tests/stand.py b/student-tests/stand.py new file mode 100755 index 0000000..3210103 --- /dev/null +++ b/student-tests/stand.py @@ -0,0 +1,138 @@ +"""Helpers for student resilience tests: the public HTTP client plus docker compose / psql / mc access. + +Standard library only (Python 3.9+). Images that need Pillow are generated inside the API container. +Never prints tokens or secrets. +""" +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +# lab7: every student test talks HTTPS to https://localhost:8443 and verifies it against the stand CA. +os.environ.setdefault("CA_FILE", str(ROOT / "secrets/pki/ca.crt")) +os.environ.setdefault("BASE_URL", "https://localhost:8443") +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402,F401 + +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def log(*parts): + print(time.strftime("%H:%M:%S"), *parts, flush=True) + + +def compose(*args, check_rc=True, capture=True, timeout=180): + done = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=capture, text=True, timeout=timeout) + if check_rc and done.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args[:2]), (done.stderr or "")[-300:])) + return done.stdout if capture else "" + + +def started_at(*services): + """Container name -> State.StartedAt, proof that a restart really happened.""" + ids = compose("ps", "-q", *services).split() + out = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids], + capture_output=True, text=True).stdout + return "; ".join(line.strip("/")[:60] for line in out.splitlines()) + + +def sql(query): + """Run a read query in postgres; returns rows as lists of strings (tab separated).""" + out = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t", "-c", query) + return [line.split("\t") for line in out.splitlines() if line] + + +def s3_keys(prefix): + """Object keys starting with prefix, listed with the root credentials inside a throwaway minio-init container.""" + wanted, prefix = prefix, prefix[:prefix.rfind("/") + 1] + script = ('mkdir -p /tmp/mc/certs/CAs && cp /run/secrets/ca.crt /tmp/mc/certs/CAs/ca.crt && ' + 'mc alias set l https://minio:9000 "$(cat /run/secrets/minio_root_user)" ' + '"$(cat /run/secrets/minio_root_password)" >/dev/null && ' + 'mc ls --recursive --json l/messenger-attachments/%s || true' % prefix) + out = compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "minio-init", "-c", script) + keys = [] + for line in out.splitlines(): + try: + item = json.loads(line) + except ValueError: + continue + if item.get("key"): + keys.append(prefix + item["key"]) + return sorted(k for k in keys if k.startswith(wanted)) + + +def queue_depth(queue="attachments.process"): + """None while the broker node is still booting (right after a restart); callers poll (wait_queue).""" + out = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "--vhost", "messenger", "list_queues", + "name", "messages", "messages_ready", "messages_unacknowledged", check_rc=False) + for line in out.splitlines(): + parts = line.split() + if parts and parts[0] == queue and len(parts) == 4 and parts[1].isdigit(): + return {"messages": int(parts[1]), "ready": int(parts[2]), "unacked": int(parts[3])} + return None + + +def wait_queue(min_ready=1, seconds=30): + """Quorum queue counters are refreshed by the stats emitter (~5 s), so poll.""" + deadline = time.monotonic() + seconds + depth = None + while time.monotonic() < deadline: + depth = queue_depth() + if depth and depth["ready"] >= min_ready: + return depth + time.sleep(1) + return depth + + +def jobs_cli(*args): + done = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "scripts/jobs.py", *args], + cwd=ROOT, capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) if done.stdout.strip() else None + + +def wait_http_ready(c, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + return + except Failure: + time.sleep(1) + raise Failure("API not ready after %ds" % seconds) + + +def upload(c, actor, chat, body=SAMPLE, filename="sample.png", ctype="image/png", expected=202): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")) + schema = "Attachment" if expected == 202 else None + return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype}, schema=schema)["json"] + + +def get(c, actor, att_id): + return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"] + + +def wait_status(c, actor, att_id, statuses, seconds=120): + deadline = time.monotonic() + seconds + seen = [] + while time.monotonic() < deadline: + att = get(c, actor, att_id) + if not seen or seen[-1] != att["status"]: + seen.append(att["status"]) + if att["status"] in statuses: + return att, seen + time.sleep(0.5) + raise Failure("attachment %s did not reach %s in %ds (seen %s)" % (att_id, statuses, seconds, seen)) + + +def objects_of(att_id): + return s3_keys("attachments/%s/" % att_id) + s3_keys("staging/%s" % att_id) + + +def unique_name(fault=None): + tag = uuid.uuid4().hex[:8] + return ("fault-%s-%s.png" % (fault, tag)) if fault else ("plain-%s.png" % tag) diff --git a/student-tests/test_auth_acl.py b/student-tests/test_auth_acl.py new file mode 100644 index 0000000..a98fb3f --- /dev/null +++ b/student-tests/test_auth_acl.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Level 3-4 negative tests of the application layer over HTTPS (CA verified, no --insecure): + + 1. outsider (eve) cannot read chat / history / attachment metadata / content, cannot post, cannot attach + another chat's attachment; forged X-User-Id never changes the Bearer identity; Basic on business endpoints = 401 + 2. logout revokes the token on every replica; an expired session is rejected on every replica + 3. login rate limit shared by the replicas: 5 failures per username -> 429 + Retry-After (even with the right + password), counted across api-1 and api-2; per-IP limit -> 429 + 4. request size limits: JSON > 64 KiB -> 413, upload > 10 MiB -> 413 + 5. Argon2id: stored hash has salt + parameters; an old-parameter hash is upgraded on the next successful login + +python3 student-tests/test_auth_acl.py (stand running; ~2 min because of the expiry and rate-limit windows) +""" +import base64 +import hashlib +import json +import time +import uuid + +from stand import SAMPLE, Client, check, compose, log, run, sql, upload, wait_status + +RESULTS = [] + + +def ok(name, detail=""): + RESULTS.append(name) + log("PASS %s %s" % (name, detail)) + + +def instances(responses): + return sorted({r["headers"].get("X-Instance-Id") for r in responses}) + + +def basic(username, password): + return {"Authorization": "Basic " + base64.b64encode(("%s:%s" % (username, password)).encode()).decode()} + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat) + wait_status(c, alice, att["id"], ("ready",), 90) + cid, aid = chat["id"], att["id"] + + # ------------------------------------------------------------------ 1. ACL + for method, path, body in (("GET", "/api/v1/chats/%s" % cid, None), ("GET", "/api/v1/chats/%s/messages" % cid, None), + ("GET", "/api/v1/attachments/%s" % aid, None), + ("GET", "/api/v1/attachments/%s/content" % aid, None), + ("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % aid, None)): + r = c.call(method, path, 404, actor=eve) + check(r["json"]["error"]["code"] == "not_found", "outsider must get a generic 404") + c.call("POST", "/api/v1/chats/%s/messages" % cid, 404, actor=eve, body={"text": "чужой"}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + own = c.call("POST", "/api/v1/chats", 201, actor=eve, body={"title": "eve", "member_ids": [eve["id"]]}, + schema="Chat")["json"] + c.call("POST", "/api/v1/chats/%s/messages" % own["id"], 404, actor=eve, body={"text": "", "attachment_ids": [aid]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + listed = c.call("GET", "/api/v1/chats", actor=eve, schema="ChatPage")["json"]["items"] + check(cid not in [x["id"] for x in listed], "foreign chat must not be listed") + ok("acl_outsider", "eve: 404 on chat/history/metadata/content/variant/post, cannot attach a foreign file") + + me = c.call("GET", "/api/v1/users/me", actor=eve, headers={"X-User-Id": alice["id"]}, schema="User")["json"] + check(me["id"] == eve["id"], "X-User-Id must not change the Bearer identity") + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + c.call("GET", "/api/v1/chats/%s/messages" % cid, 401, headers=basic(alice["user"]["username"], + alice["registration"]["password"])) + ok("identity_spoofing", "forged X-User-Id ignored; without Bearer 401; Basic on business endpoint 401") + + # ------------------------------------------------------------------ 2. logout and expiry on every replica + s = c.user("logout") + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=s) + seen = [c.call("GET", "/api/v1/users/me", 401, actor=s) for _ in range(6)] + check(len(instances(seen)) == 2, "revocation must be checked on both replicas, saw %s" % instances(seen)) + ok("logout", "revoked token: 401 on %s" % instances(seen)) + + x = c.user("expiry") + c.call("GET", "/api/v1/users/me", actor=x, schema="User") + digest = hashlib.sha256(x["token"].encode()).hexdigest() + sql("UPDATE sessions SET expires_at = now() - interval '1 second' WHERE token_hash = '\\x%s'" % digest) + log("session expires_at moved to the past in PostgreSQL; the Valkey cache entry lives at most 30 s") + time.sleep(32) + seen = [c.call("GET", "/api/v1/users/me", 401, actor=x) for _ in range(6)] + ok("expiry", "expired session: 401 on %s after the 30 s cache bound" % instances(seen)) + + # ------------------------------------------------------------------ 5. Argon2id parameters and rehash + r = c.user("argon") + row = sql("SELECT password_hash FROM users WHERE id = '%s'" % r["id"])[0][0] + check(row.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and row.count("$") == 5, "PHC string with params + salt") + weak = compose("exec", "-T", "api-1", "python", "-c", + "from argon2 import PasswordHasher;import sys;" + "print(PasswordHasher(time_cost=1,memory_cost=8192,parallelism=1).hash(sys.argv[1]))", + r["registration"]["password"]).strip() + sql("UPDATE users SET password_hash = '%s' WHERE id = '%s'" % (weak, r["id"])) + username, password = r["user"]["username"], r["registration"]["password"] + c.call("POST", "/api/v1/auth/sessions", 201, headers=basic(username, password), schema="Session") + upgraded = sql("SELECT password_hash FROM users WHERE id = '%s'" % r["id"])[0][0] + check(upgraded.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and upgraded != weak, "hash must be upgraded") + ok("rehash", "stored %s -> %s after a successful login" % (weak.split("$")[3], upgraded.split("$")[3])) + + # ------------------------------------------------------------------ 4. size limits + big = {"text": "x" * 70000} + r413 = c.call("POST", "/api/v1/chats/%s/messages" % cid, 413, actor=alice, body=big, + headers={"Idempotency-Key": uuid.uuid4().hex}) + c.call("POST", "/api/v1/chats/%s/attachments?filename=big.png" % cid, 413, actor=alice, + raw=SAMPLE + b"\0" * (10 * 1024 * 1024), headers={"Content-Type": "image/png"}) + ok("size_limits", "JSON 70 kB -> 413 (%s), upload > 10 MiB -> 413" % r413["json"]["error"]["code"]) + + # ------------------------------------------------------------------ 3. shared login rate limit + victim = c.user("brute") + name = victim["user"]["username"] + fails = [c.call("POST", "/api/v1/auth/sessions", 401, headers=basic(name, "wrong-password-123")) for _ in range(5)] + limited = c.call("POST", "/api/v1/auth/sessions", 429, headers=basic(name, victim["registration"]["password"])) + retry = int(limited["headers"]["Retry-After"]) + check(limited["json"]["error"]["code"] == "rate_limited" and 1 <= retry <= 60, "429 rate_limited + Retry-After") + check(len(instances(fails + [limited])) == 2, "the 6 attempts must hit both replicas") + body = json.dumps(limited["json"], ensure_ascii=False) + check(victim["registration"]["password"] not in body and "argon" not in body.lower(), "no credential detail") + ok("rate_limit_user", "5 failures on %s -> 429 Retry-After=%ds even with the right password" % ( + instances(fails + [limited]), retry)) + + statuses = [] + for i in range(130): + statuses.append(c.call("POST", "/api/v1/auth/sessions", [401, 429], + headers=basic("nouser_%d" % i, "wrong-password-123"))["status"]) + if statuses[-1] == 429: + break + check(statuses[-1] == 429, "per-IP limit must trigger within 130 attempts") + ok("rate_limit_ip", "per-IP limit: 429 after %d attempts from one address" % len(statuses)) + log("waiting for the rate-limit window to expire so that later tests are not affected") + time.sleep(61) + print("PASS: %s" % ", ".join(RESULTS)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_bao_sealed.py b/student-tests/test_bao_sealed.py new file mode 100644 index 0000000..c0f39f1 --- /dev/null +++ b/student-tests/test_bao_sealed.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Level 4: bounded behaviour while OpenBao is sealed (operator incident drill). + + 1. operator seals OpenBao (scripts/bao.sh seal) + 2. running API/worker keep the secrets they hold: public smoke + an upload pipeline still pass; + the refresher logs openbao_unavailable (no crash, no fallback to anything weaker) + 3. a worker restarted while sealed does NOT start with missing/empty credentials: it waits (openbao_not_ready), + stays unhealthy, and the other worker keeps processing + 4. operator unseals (scripts/bao.sh unseal): the waiting worker logs in, becomes healthy; times are recorded +python3 student-tests/test_bao_sealed.py (needs the operator OpenBao material) +""" +import subprocess +import time + +from stand import ROOT, Client, check, compose, log, run, upload, wait_status + +WORKER2 = "mtusi-lab7-worker-2" + + +def bao(cmd): + out = subprocess.run(["sh", "scripts/bao.sh", cmd], cwd=ROOT, capture_output=True, text=True) + check(out.returncode == 0, "bao.sh %s failed: %s" % (cmd, out.stderr[-200:])) + return out.stdout.strip().splitlines()[-1] + + +def health(container): + return subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", container], + capture_output=True, text=True).stdout.strip() + + +def main(): + c = Client() + alice, _bob, _eve, chat = c.bootstrap() + t0 = time.time() + log(bao("seal")) + try: + status = compose("exec", "-T", "-e", "BAO_ADDR=https://127.0.0.1:8200", "-e", "BAO_CACERT=/run/secrets/ca.crt", + "openbao", "bao", "status", check_rc=False) + check("Sealed true" in status, "OpenBao must be sealed") + smoke = subprocess.run(["make", "test", "CA_FILE=%s" % (ROOT / "secrets/pki/ca.crt")], cwd=ROOT, + capture_output=True, text=True) + check(smoke.returncode == 0, "public smoke must pass while OpenBao is sealed") + log("sealed: running services keep working -> %s" % smoke.stdout.strip().splitlines()[-1]) + time.sleep(20) + logs = compose("logs", "--since", "60s", "api-1", "worker") + check("openbao_unavailable" in logs, "refresher must report the sealed OpenBao") + log("refresher: openbao_unavailable ... keeping cached secret version (seen in API/worker logs)") + + subprocess.run(["docker", "restart", "-t", "20", WORKER2], capture_output=True, check=True) + time.sleep(35) + state = health(WORKER2) + wlogs = subprocess.run(["docker", "logs", "--since", "40s", WORKER2], capture_output=True, text=True) + waiting = "openbao_not_ready" in (wlogs.stdout + wlogs.stderr) + log("worker-2 restarted while sealed: health=%s, waiting for OpenBao=%s" % (state, waiting)) + check(state != "healthy" and waiting, "a restarted worker must wait, not run without credentials") + att = upload(c, alice, chat, filename="sealed.png") + wait_status(c, alice, att["id"], ("ready",), 90) + log("upload during the seal processed by the remaining worker: ready") + finally: + t1 = time.time() + log(bao("unseal")) + deadline = time.time() + 90 + while time.time() < deadline and health(WORKER2) != "healthy": + time.sleep(2) + check(health(WORKER2) == "healthy", "worker-2 must recover after unseal") + log("worker-2 healthy %.0f s after unseal (sealed for %.0f s in total)" % (time.time() - t1, t1 - t0)) + print("PASS: sealed OpenBao -> running services continue, new process waits, recovery after unseal") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_cert_rotation.py b/student-tests/test_cert_rotation.py new file mode 100644 index 0000000..f5454ff --- /dev/null +++ b/student-tests/test_cert_rotation.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Level 4: rotation of a leaf certificate WITHOUT rebuilding images, revocation of the old identity, expiry. + +Leaf: lb-client (identity lb.client.messenger), the only client allowed by the API replicas. + 1. background HTTPS traffic through nginx for the whole test (errors are counted) + 2. scripts/pki.sh issue lb-client new key + serial, files overwritten in place (bind-mounted secrets) + 3. nginx -s reload new upstream connections present the new certificate; no image build, + no container recreate + 4. old and new certificate both still accepted (old is valid until revoked) + 5. scripts/pki.sh revoke-serial appended to revoked-serials.txt; the API re-reads it within ~1 s + 6. old certificate -> refused (reason=revoked in the API log), new -> 200, expired copy -> refused by TLS + 7. public smoke afterwards +New connections are what proves revocation: established keep-alive/TLS sessions are not re-checked, so nginx's +upstream keep-alive (4 s) is the upper bound for an already open connection of the old identity. + +python3 student-tests/test_cert_rotation.py +""" +import json +import shutil +import subprocess +import threading +import time + +from openloop import Load +from stand import ROOT, check, compose, log, run + +PKI = ROOT / "secrets/pki" + + +def sh(*args): + return subprocess.run(["sh", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + + +def probe(host, *certs): + out = compose("run", "--rm", "--no-deps", "-v", "./secrets/pki:/pki:ro", "-v", "./student-tests:/audit:ro", + "--entrypoint", "python", "worker", "/audit/mtls_probe.py", host, *certs) + return {r["cert"]: r for r in (json.loads(line) for line in out.splitlines() if line.startswith("{"))} + + +def main(): + load = Load("https://localhost:8443", users=2, threads=32) + stop, summary = threading.Event(), {} + thread = threading.Thread(target=lambda: summary.update(load.run(10, 900, stop_event=stop)), daemon=True) + thread.start() + time.sleep(5) + old = sh("scripts/pki.sh", "serial", "lb-client") + log("current lb-client serial %s" % old) + t0 = time.time() + log(sh("scripts/pki.sh", "issue", "lb-client").splitlines()[0]) + shutil.copy(PKI / "lb-client.crt.previous", PKI / "lb-client-old.crt") + shutil.copy(PKI / "lb-client.key.previous", PKI / "lb-client-old.key") + new = sh("scripts/pki.sh", "serial", "lb-client") + check(new != old, "a new serial must be issued") + compose("exec", "-T", "nginx", "nginx", "-s", "reload") + log("nginx reloaded with the new client certificate (%.1f s after issue)" % (time.time() - t0)) + if not (PKI / "lb-client-expired.crt").exists(): + sh("scripts/pki.sh", "expired", "lb-client") + before = probe("api-1", "lb-client-old", "lb-client") + check(before["lb-client-old"]["verdict"] == "accepted" and before["lb-client"]["verdict"] == "accepted", + "both certificates valid before revocation: %s" % before) + log("before revocation: old=%s new=%s" % (before["lb-client-old"]["detail"], before["lb-client"]["detail"])) + log(sh("scripts/pki.sh", "revoke-serial", old)) + time.sleep(2) + for api in ("api-1", "api-2"): + after = probe(api, "lb-client-old", "lb-client", "lb-client-expired") + check(after["lb-client-old"]["verdict"] == "rejected", "%s: revoked old cert must be refused" % api) + check(after["lb-client"]["verdict"] == "accepted", "%s: new cert must work" % api) + check(after["lb-client-expired"]["verdict"] == "rejected", "%s: expired cert must be refused" % api) + log("%s after revocation: old -> %s | new -> %s | expired -> %s" % ( + api, after["lb-client-old"]["detail"], after["lb-client"]["detail"], after["lb-client-expired"]["detail"])) + logs = compose("logs", "--since", "2m", "api-1", "api-2") + check("reason=revoked" in logs and old in logs.upper(), "API must log the revoked serial") + log("API log: mtls_client_rejected reason=revoked serial=%s" % old) + time.sleep(5) + stop.set() + thread.join(60) + bad = summary.get("server_errors_unexpected") + log("traffic during rotation: %s requests, %s ok, unexpected errors %s" % ( + summary.get("requests"), summary.get("ok"), bad)) + check(bad == 0, "rotation must not break traffic") + out = subprocess.run(["make", "test", "CA_FILE=%s" % (PKI / "ca.crt")], cwd=ROOT, capture_output=True, text=True) + check(out.returncode == 0, "smoke after rotation") + log(out.stdout.strip().splitlines()[-1]) + print("PASS: lb-client rotated (%s -> %s) without image rebuild, old serial revoked, expired cert refused" % (old, new)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_exporter_queue_full.py b/student-tests/test_exporter_queue_full.py new file mode 100644 index 0000000..504c287 --- /dev/null +++ b/student-tests/test_exporter_queue_full.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Bounded buffering at the limit: Tempo stopped under 100 req/s until the exporter queue is FULL. +Expected: queue_size stops at capacity, extra spans are dropped and counted, collector RSS stays under the container +limit, API has no unexpected errors. python3 student-tests/test_exporter_queue_full.py (~5 min) +""" +import threading +import time + +import obs +from openloop import Load +from stand import check, compose, log, run +from test_incidents import docker_mem + + +def value(expr): + return obs.prom_scalar("sum(%s)" % expr) + + +def main(): + load = Load("https://localhost:8443", users=4, threads=96) + stop, summary = threading.Event(), {} + thread = threading.Thread(target=lambda: summary.update(load.run(100, 900, 1, stop_event=stop)), daemon=True) + thread.start() + time.sleep(20) + q = 'otelcol_exporter_queue_size{exporter="otlp/tempo"}' + capacity = value('otelcol_exporter_queue_capacity{exporter="otlp/tempo"}') + dropped0, refused0 = value("otelcol_exporter_enqueue_failed_spans_total"), value("otelcol_receiver_refused_spans_total") + compose("stop", "-t", "5", "tempo") + t0 = time.time() + log("tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=%.0f batches" % capacity) + peak_q, mems = 0, [] + try: + for _ in range(16): + time.sleep(12) + size, mem = value(q), docker_mem("otel-collector")[0] + peak_q, _ = max(peak_q, size), mems.append(mem) + log(" +%3d s queue=%.0f/%.0f collector_rss=%s dropped=%.0f refused=%.0f" % ( + time.time() - t0, size, capacity, mem, value("otelcol_exporter_enqueue_failed_spans_total") - dropped0, + value("otelcol_receiver_refused_spans_total") - refused0)) + dropped = value("otelcol_exporter_enqueue_failed_spans_total") - dropped0 + refused = value("otelcol_receiver_refused_spans_total") - refused0 + finally: + compose("start", "tempo") + stop.set() + thread.join(60) + log("result: peak queue=%.0f/%.0f, dropped by full queue=%.0f spans, refused by memory_limiter=%.0f spans, peak RSS=%s MiB" + % (peak_q, capacity, dropped, refused, max(float(m.replace("MiB", "")) for m in mems))) + log("API: requests=%s ok=%s unexpected_errors=%s p99=%s ms" % ( + summary["requests"], summary["ok"], summary["server_errors_unexpected"], summary["latency_ms_plain_ok"]["p99"])) + check(peak_q <= capacity, "queue must stay within capacity") + check(peak_q >= capacity * 0.95 or refused > 0, "the limit must really be reached") + check(dropped + refused > 0, "overflow must be counted") + check(summary["server_errors_unexpected"] == 0, "API must be unaffected") + print("PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_failure_windows.py b/student-tests/test_failure_windows.py new file mode 100755 index 0000000..93f59a0 --- /dev/null +++ b/student-tests/test_failure_windows.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Grade 4 fault-injection suite (needs FAULT_INJECTION=1 on the stand, see REPORT.md). + +Each scenario uploads a file whose display name selects an injected fault (app/faults.py) and checks the +recovery through the public API plus DB/S3/broker state: + A api_crash_after_commit crash between COMMIT(job+outbox) and publish -> another replica's relay publishes + B worker_crash_after_put crash after S3 writes, before result COMMIT -> lease expiry -> same keys, one result + C worker_crash_after_commit crash after COMMIT, before ack -> redelivery is a no-op + D transient3 3 transient errors -> retries with growing jittered backoff -> ready + E permanent permanent error -> failed at attempt 1, no retry, content/attach 409 + F transient6 attempts exhausted -> failed/retry_exhausted -> registry -> explicit replay -> ready; + replay of a ready job is refused + G worker_hang hung worker (no heartbeats) -> other worker takes over, hung one is fenced + H poison message invalid broker message -> DLQ, not retried forever + I broker down RabbitMQ stopped: upload still 202 (job waits in outbox) -> broker back -> ready +usage: python3 student-tests/test_failure_windows.py [A B C ...] +""" +import sys +import time + +from stand import (Client, Failure, check, compose, get, jobs_cli, log, objects_of, queue_depth, run, sql, upload, + unique_name, wait_http_ready, wait_status) + + +def audit(att_id): + return sql("SELECT action, actor, attempt, coalesce(detail->>'error_code',''), coalesce(detail->>'delay_seconds','')" + " FROM job_audit WHERE attachment_id='%s' ORDER BY id" % att_id) + + +def show_audit(att_id): + for action, actor, attempt, code, delay in audit(att_id): + log(" audit attempt=%s %-16s %-28s %s %s" % (attempt, action, actor[:28], code, ("delay=%ss" % delay) if delay else "")) + + +def one_result(att_id): + variants = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att_id)[0][0] + succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att_id)[0][0] + keys = objects_of(att_id) + check(variants == "3", "exactly one set of 3 variant rows, got %s" % variants) + check(succeeded == "1", "exactly one committed success, got %s" % succeeded) + check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), + "original + 3 variants, staging cleaned: %s" % keys) + + +def ready_and_single(c, actor, att_id, seconds=120): + att, seen = wait_status(c, actor, att_id, ("ready", "failed"), seconds) + check(att["status"] == "ready", "expected ready, got %s/%s" % (att["status"], att["error_code"])) + log(" statuses:", " -> ".join(seen)) + show_audit(att_id) + one_result(att_id) + return att + + +def scenario_a(c, alice, bob, chat): + name = unique_name("api_crash_after_commit") + try: + upload(c, alice, chat, filename=name, expected=[202, 500, 502, 503]) + except Failure as exc: + log(" client saw:", exc) + wait_http_ready(c) + rows = sql("SELECT id FROM attachments WHERE filename='%s'" % name) + check(len(rows) == 1, "committed attachment must exist after API crash") + att_id = rows[0][0] + log(" committed attachment %s survived the API crash" % att_id) + ready_and_single(c, bob, att_id) + pub = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att_id)[0][0] + check(pub == "1", "outbox row published exactly once by the relay") + + +def scenario_b(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_crash_after_put")) + ready_and_single(c, bob, att["id"]) + actions = [r[0] for r in audit(att["id"])] + check("retry_scheduled" in actions, "lease expiry must schedule a new attempt") + + +def scenario_c(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_crash_after_commit")) + ready_and_single(c, bob, att["id"]) + time.sleep(8) + logs = compose("logs", "--since", "60s", "worker") + check("decision=done" in logs, "redelivered message after crash must be a no-op") + one_result(att["id"]) + + +def scenario_d(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("transient3")) + ready_and_single(c, bob, att["id"]) + delays = [float(r[4]) for r in audit(att["id"]) if r[0] == "retry_scheduled"] + check(len(delays) == 3, "three retries expected, got %s" % delays) + bounds = [(1, 2), (2, 4), (4, 8)] + check(all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "equal-jitter backoff bounds %s" % delays) + log(" backoff delays:", delays) + + +def scenario_e(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("permanent")) + failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", "permanent -> failed/invalid_image") + check(failed["variants"] == [], "failed attachment has no variants") + show_audit(att["id"]) + check([r[0] for r in audit(att["id"])].count("claimed") == 1, "permanent errors are not retried") + c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "k-" + att["id"]}) + + +def scenario_f(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("transient6")) + failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "exhausted -> retry_exhausted") + attempts = sql("SELECT attempt, max_attempts FROM attachment_jobs WHERE attachment_id='%s'" % att["id"])[0] + check(attempts == ["5", "5"], "bounded: stopped at attempt 5 of 5, got %s" % attempts) + rc, registry = jobs_cli("failed") + check(any(r["attachment_id"] == att["id"] for r in registry), "failed job listed in operator registry") + log(" failed registry contains the job; replay") + rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite") + check(rc == 0 and result["result"] == "queued", "replay must queue: %s" % result) + ready_and_single(c, bob, att["id"], 120) + rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite") + check(rc == 3 and result["result"] == "refused", "replay of a ready job must be refused: %s" % result) + log(" second replay refused:", result["reason"]) + one_result(att["id"]) + + +def scenario_g(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_hang")) + ready_and_single(c, bob, att["id"], 150) + rows = audit(att["id"]) + check(any(r[0] == "retry_scheduled" and r[3] == "lease_expired" for r in rows), "lease expiry detected") + time.sleep(30) # the hung worker wakes up and must be fenced + logs = compose("logs", "--since", "180s", "worker") + check("lease_lost" in logs or "job_result_not_committed" in logs, "hung worker must be fenced by the lease") + one_result(att["id"]) + + +POISON = ("import asyncio, aio_pika\n" + "from app import config, broker\n" + "async def m():\n" + " conn = await broker.connect(); ch = await conn.channel(publisher_confirms=True)\n" + " ex = await broker.get_exchange(ch)\n" + " await ex.publish(aio_pika.Message(b'{not json', delivery_mode=2), routing_key=broker.ROUTING_KEY)\n" + " await conn.close()\n" + "asyncio.run(m())\n") + + +def scenario_h(c, alice, bob, chat): + before = (queue_depth("attachments.dlq") or {}).get("messages", 0) + compose("exec", "-T", "api-1", "python", "-c", POISON) + deadline = time.monotonic() + 30 + after = before + while time.monotonic() < deadline and after <= before: + time.sleep(1) + after = (queue_depth("attachments.dlq") or {}).get("messages", 0) + check(after == before + 1, "poison message must land in attachments.dlq (%s -> %s)" % (before, after)) + log(" DLQ depth %s -> %s" % (before, after)) + + +def scenario_i(c, alice, bob, chat): + compose("stop", "-t", "10", "rabbitmq") + try: + att = upload(c, alice, chat, filename=unique_name()) + check(att["status"] == "queued", "upload accepted while broker is down") + c.message(alice, chat, "", attachments=[att["id"]]) + time.sleep(3) + pending = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0] + check(pending == "1", "job must wait in the outbox, got %s" % pending) + check(get(c, bob, att["id"])["status"] == "queued", "still queued while broker is down") + log(" broker down: upload 202, outbox holds 1 unpublished job") + finally: + compose("start", "rabbitmq") + ready_and_single(c, bob, att["id"], 120) + + +SCENARIOS = {"A": scenario_a, "B": scenario_b, "C": scenario_c, "D": scenario_d, "E": scenario_e, + "F": scenario_f, "G": scenario_g, "H": scenario_h, "I": scenario_i} + + +def main(): + c = Client(timeout=30) + alice, bob, eve, chat = c.bootstrap() + picked = sys.argv[1:] or list(SCENARIOS) + for key in picked: + log("scenario %s: %s" % (key, SCENARIOS[key].__name__)) + started = time.monotonic() + SCENARIOS[key](c, alice, bob, chat) + log("scenario %s PASS (%.1fs)" % (key, time.monotonic() - started)) + print("PASS: failure windows %s" % " ".join(picked)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_incidents.py b/student-tests/test_incidents.py new file mode 100644 index 0000000..b9235fb --- /dev/null +++ b/student-tests/test_incidents.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Level 4 controlled incidents, diagnosed ONLY from stored telemetry (Prometheus / Loki / Tempo), with alert +firing -> resolved timings. A background open-loop traffic (messages + history, 2% slow handler requests) runs all +the time so that error ratios have a denominator. + + A. worker stopped -> MessengerQueueLagGrowing firing; start worker -> backlog drains -> resolved + B. DB table lock (slow DB) -> MessengerApiHighErrorRate firing; trace shows the INSERT span waiting 5 s on the lock + (vs. an X-Fault: slow request: long handler, short DB spans); unlock -> resolved + C. Tempo + Loki stopped -> API keeps working, collector queue bounded, drops counted, TelemetryExportFailing; + collector stopped -> API keeps working, SDK queues bounded (RSS flat) + +python3 student-tests/test_incidents.py [--only A,B,C] (stack running, FAULT_INJECTION=1; ~15 min) +""" +import argparse +import json +import subprocess +import threading +import time +from pathlib import Path + +import obs +from openloop import Load +from stand import Client, check, compose, log, run, upload, wait_status + +ROOT = Path(__file__).resolve().parents[1] +RESULTS = {} + + +def docker_mem(service): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}"], + capture_output=True, text=True).stdout + for line in out.splitlines(): + name, mem, cpu = line.split("\t") + if name.startswith("mtusi-lab6-%s-" % service): + return mem.split(" / ")[0], cpu + return "stopped", "-" + + +def wait_alert(name, want, seconds, t0): + """want='firing' waits for firing; want='resolved' waits until the alert is gone. Returns seconds since t0.""" + deadline = time.monotonic() + seconds + pending_at = None + while time.monotonic() < deadline: + state = obs.alerts().get(name) + if state == "pending" and pending_at is None: + pending_at = time.time() - t0 + log(" %s pending (+%.0f s)" % (name, pending_at)) + if (want == "firing" and state == "firing") or (want == "resolved" and state is None): + elapsed = time.time() - t0 + log(" %s %s (+%.0f s)" % (name, want, elapsed)) + return elapsed, pending_at + time.sleep(5) + raise AssertionError("%s did not become %s within %d s" % (name, want, seconds)) + + +def psql(query): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query) + + +# ------------------------------------------------------------------------------------------------ A +def incident_worker(c, alice, chat): + log("=== A. worker stopped (queue lag)") + t0 = time.time() + compose("stop", "-t", "20", "worker") + log("worker stopped at %s" % time.strftime("%H:%M:%S")) + atts = [upload(c, alice, chat, filename="incident-a.png") for _ in range(3)] + log("3 uploads accepted (202) while worker is down: %s" % [a["id"][:8] for a in atts]) + firing, pending = wait_alert("MessengerQueueLagGrowing", "firing", 300, t0) + + log("diagnosis from telemetry:") + age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})') + depth = {m["state"]: v for m, v in obs.prom('max by (state) (messenger_queue_depth{queue="attachments.process"})')} + processed = obs.prom_scalar('sum(rate(messenger_jobs_processed_total[1m]))') + log(" metrics: oldest job age=%.0f s, depth=%s, worker jobs/s over 1m=%.3f" % (age, depth, processed)) + since = int((time.time() - t0) / 60) + 1 + worker_lines = obs.loki('{service_name="messenger-worker"} | event=~"job_.*"', minutes=since) + worker_lines = [l for l in worker_lines if l["ts"] / 1e9 > t0 + 25] + stop_lines = obs.loki('{service_name="messenger-worker"} | event=~"worker_stop.*"', minutes=since + 1) + log(" logs: worker job_* events after stop: %d; last worker events: %s" % ( + len(worker_lines), [l.get("event") for l in stop_lines[-2:]])) + traces = obs.tempo_search('{span.messenger.attachment_id="%s"}' % atts[0]["id"], minutes=since + 1) + spans = obs.tempo_trace(traces[0]["traceID"]) if traces else [] + kinds = sorted({s["kind"] for s in spans if s["kind"] != "CLIENT"}) + log(" trace of upload %s: span kinds %s -> published, no CONSUMER: the job waits in the queue" % ( + atts[0]["id"][:8], kinds)) + check("PRODUCER" in kinds and "CONSUMER" not in kinds, "trace must stop at the producer while worker is down") + + t1 = time.time() + compose("start", "worker") + log("worker started (+%.0f s)" % (t1 - t0)) + for att in atts: + wait_status(c, alice, att["id"], ("ready",), 120) + drained = time.time() - t1 + log("backlog drained: all 3 ready %.1f s after worker start" % drained) + resolved, _ = wait_alert("MessengerQueueLagGrowing", "resolved", 300, t1) + RESULTS["A_worker_stopped"] = {"alert_pending_s": pending, "alert_firing_s": round(firing), "age_at_firing_s": + round(age), "drain_after_restart_s": round(drained, 1), + "resolved_after_restart_s": round(resolved), "lost_jobs": 0} + + +# ------------------------------------------------------------------------------------------------ B +def incident_db(c): + log("=== B. DB lock on messages (slow/unavailable DB for writes and history reads)") + t0 = time.time() + locker = subprocess.Popen( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", + "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;"], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + log("ACCESS EXCLUSIVE lock taken on table messages at %s" % time.strftime("%H:%M:%S")) + try: + firing, pending = wait_alert("MessengerApiHighErrorRate", "firing", 300, t0) + log("diagnosis from telemetry:") + ratio = obs.prom_scalar("messenger:http_error_ratio:rate2m") + p95 = obs.prom_scalar("messenger:http_latency_p95:rate5m") + by_route = obs.prom('sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count' + '{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))') + waiting = obs.prom_scalar("max(db_client_connection_pending_requests)") + used = obs.prom_scalar('max(db_client_connection_count{service_name="messenger-api",db_client_connection_state="used"})') + log(" metrics: 5xx ratio=%.1f%%, p95=%.2f s, pool used(max replica)=%.0f waiting=%.0f" % ( + ratio * 100, p95, used, waiting)) + for m, v in by_route: + log(" 5xx/s %-40s %s %.2f" % (m.get("http_route"), m.get("http_response_status_code"), v)) + db_err = obs.tempo_search('{resource.service.name="messenger-api" && span.db.operation.name="INSERT"' + ' && status=error}', minutes=5, limit=5) + check(db_err, "no trace with a failed DB span") + spans = obs.tempo_trace(db_err[0]["traceID"]) + log(" trace %s (failed request):" % db_err[0]["traceID"]) + obs.print_tree(spans, out=lambda line: print(" " + line)) + raw = obs.tempo_raw(db_err[0]["traceID"]) + cause = next((name for name in ("LockNotAvailable", "QueryCanceled", "PoolTimeout") if name in raw), "?") + db_span = max((s for s in spans if s["attrs"].get("db.system") == "postgresql"), key=lambda s: s["duration_ms"]) + server = [s for s in spans if s["kind"] == "SERVER"][0] + log(" => DB span %s took %.0f ms of %.0f ms request, status %s, exception %s: the time is spent in PostgreSQL" + % (db_span["name"], db_span["duration_ms"], server["duration_ms"], db_span["status"], cause)) + slow = obs.tempo_search('{span.messenger.fault="slow" && span.http.route="/api/v1/chats/{chat_id}/messages"}', + minutes=10, limit=50) + contrast = None + for item in slow: + sp = obs.tempo_trace(item["traceID"]) + srv = [s for s in sp if s["kind"] == "SERVER"] + dbs = [s["duration_ms"] for s in sp if s["attrs"].get("db.system") == "postgresql"] + if srv and dbs and srv[0]["attrs"].get("http.response.status_code") in ("200", 200) and max(dbs) < 100: + contrast = (item["traceID"], srv[0]["duration_ms"], max(dbs), sum(dbs)) + break + if contrast: + log(" contrast: slow HANDLER trace %s: request %.0f ms, longest DB span %.1f ms, all DB %.1f ms" + % contrast) + errors = obs.loki('{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"', minutes=5) + log(" logs: %d API warning/error records in 5 min, events=%s" % ( + len(errors), sorted({l.get("event") for l in errors}))) + blocker = psql("SELECT pid, state, wait_event_type, left(query, 60) FROM pg_stat_activity " + "WHERE query LIKE '%LOCK TABLE messages%' AND pid <> pg_backend_pid()").strip() + log(" operator step (pg_stat_activity) finds the blocker: %s" % blocker) + finally: + t1 = time.time() + psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%LOCK TABLE messages%'" + " AND pid <> pg_backend_pid()") + locker.wait(30) + log("lock released (+%.0f s)" % (t1 - t0)) + resolved, _ = wait_alert("MessengerApiHighErrorRate", "resolved", 400, t1) + RESULTS["B_db_lock"] = {"alert_pending_s": pending, "alert_firing_s": round(firing), + "error_ratio_at_firing": round(ratio, 3), "p95_at_firing_s": round(p95, 2), + "db_span_ms": round(db_span["duration_ms"]), "request_ms": round(server["duration_ms"]), + "cause": cause, "resolved_after_unlock_s": round(resolved)} + + +# ------------------------------------------------------------------------------------------------ C +def counter(expr): + return obs.prom_scalar("sum(%s)" % expr) + + +def incident_telemetry(load): + log("=== C1. Tempo and Loki stopped (telemetry backends down)") + before_res = len(load.results) + dropped0 = counter('otelcol_exporter_enqueue_failed_spans_total') + counter( + 'otelcol_exporter_enqueue_failed_log_records_total') + failed0 = counter('otelcol_exporter_send_failed_spans_total') + counter( + 'otelcol_exporter_send_failed_log_records_total') + t0 = time.time() + compose("stop", "-t", "5", "tempo", "loki") + samples = [] + for i in range(14): # ~140 s + time.sleep(10) + mem, cpu = docker_mem("otel-collector") + q_tempo = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlp/tempo"})') + q_loki = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})') + samples.append({"t": round(time.time() - t0), "collector_mem": mem, "cpu": cpu, "queue_tempo": q_tempo, + "queue_loki": q_loki}) + log(" +%3d s collector mem=%s cpu=%s queue tempo=%.0f loki=%.0f (capacity 100 batches)" % ( + samples[-1]["t"], mem, cpu, q_tempo, q_loki)) + alert_state = obs.alerts().get("TelemetryExportFailing") + log(" alert TelemetryExportFailing during outage: %s" % alert_state) + check(alert_state == "firing", "TelemetryExportFailing must fire while backends are down") + window = [r for r in load.results[before_res:]] + bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)] + log(" API during the outage: %d requests, unexpected errors=%d; alert TelemetryExportFailing=%s" % ( + len(window), len(bad), alert_state)) + check(not bad, "API must keep working while telemetry backends are down") + compose("start", "tempo", "loki") + obs.wait_for(lambda: subprocess.run(["curl", "-sf", obs.TEMPO + "/ready"], capture_output=True).returncode == 0 + and subprocess.run(["curl", "-sf", obs.LOKI + "/ready"], capture_output=True).returncode == 0, 120, 3) + t1 = time.time() + log(" tempo/loki started and ready (+%.0f s)" % (t1 - t0)) + fresh = obs.wait_for(lambda: obs.tempo_search('{resource.service.name="messenger-api"}', start=t1, + end=time.time() + 5, limit=20), 90, 5) + log(" new traces after restart visible: %d (export resumed)" % len(fresh or [])) + check(fresh, "traces must flow again after the backends are back") + resolved, _ = wait_alert("TelemetryExportFailing", "resolved", 300, t1) + time.sleep(20) + dropped = counter('otelcol_exporter_enqueue_failed_spans_total') + counter( + 'otelcol_exporter_enqueue_failed_log_records_total') - dropped0 + failed = counter('otelcol_exporter_send_failed_spans_total') + counter( + 'otelcol_exporter_send_failed_log_records_total') - failed0 + log(" telemetry lost and counted: enqueue_failed(queue full)=%.0f items, send_failed(retries exhausted)=%.0f items" + % (dropped, failed)) + RESULTS["C1_backends_down"] = {"outage_s": round(t1 - t0), "api_requests": len(window), "api_unexpected_errors": + len(bad), "collector_samples": samples, "alert": alert_state, + "dropped_items": dropped, "send_failed_items": failed, + "traces_after_restart": len(fresh or []), + "alert_resolved_after_restart_s": round(resolved)} + + log("=== C2. Collector stopped (SDK side buffering)") + before_res = len(load.results) + api_before = docker_mem("api-1")[0] + compose("stop", "-t", "5", "otel-collector") + mems = [] + for _ in range(6): + time.sleep(10) + mems.append(docker_mem("api-1")[0]) + window = load.results[before_res:] + bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)] + compose("start", "otel-collector") + log(" api-1 RSS before=%s during=%s; %d requests, unexpected errors=%d" % (api_before, mems, len(window), len(bad))) + check(not bad, "API must keep working while the collector is down") + RESULTS["C2_collector_down"] = {"outage_s": 60, "api1_mem_before": api_before, "api1_mem_during": mems, + "api_requests": len(window), "api_unexpected_errors": len(bad)} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--only", default="A,B,C") + p.add_argument("--output", default=str(ROOT / "evidence/incidents.json")) + args = p.parse_args() + c = Client() + alice, _bob, _eve, chat = c.bootstrap() + load = Load("https://localhost:8443", users=2, threads=48) + stop = threading.Event() + traffic = threading.Thread(target=lambda: RESULTS.setdefault("traffic", load.run( + 10, 3600, 0, slow_pct=2, error_pct=0, stop_event=stop)), daemon=True) + traffic.start() + log("background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow)") + time.sleep(30) + try: + if "A" in args.only: + incident_worker(c, alice, chat) + if "B" in args.only: + incident_db(c) + if "C" in args.only: + incident_telemetry(load) + finally: + stop.set() + traffic.join(60) + Path(args.output).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2, default=str) + "\n") + log("summary written to %s" % args.output) + print("PASS: incidents %s diagnosed from telemetry, alerts fired and resolved" % args.only) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_observability.py b/student-tests/test_observability.py new file mode 100644 index 0000000..3ff159a --- /dev/null +++ b/student-tests/test_observability.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Levels 3-4: one controlled upload and one failed request are found in traces, logs and metrics; privacy checks. + + 1. upload with fault-transient1 (attempt 1 fails transiently, attempt 2 succeeds) -> ONE trace: + API SERVER span -> PRODUCER -> CONSUMER (retry, ERROR) -> PRODUCER -> CONSUMER (succeeded) -> S3/DB spans + 2. the same trace_id in Loki for messenger-api and messenger-worker log records + 3. X-Request-Id of a 500 (X-Fault: error) -> Loki log line -> its trace_id -> Tempo server span with that request_id + 4. no password / token / message text / filename in logs or traces; no ids in metric labels + 5. all metric names used by the dashboard and alert rules exist and have samples + +python3 student-tests/test_observability.py (stack from REPORT.md must be running; FAULT_INJECTION=1) +""" +import re +import time +import uuid + +import obs +from stand import Client, check, log, run, unique_name, upload, wait_status + +UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url_full", "url_query", + "filename", "text", "http_target", "url_path"} +REQUIRED_METRICS = [ + "http_server_request_duration_seconds_bucket", "messenger_jobs_processed_total", + "messenger_job_duration_seconds_bucket", "messenger_attachment_time_to_ready_seconds_bucket", + "messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_worker_busy", + "messenger_worker_capacity", "db_client_connection_count", "db_client_connection_pending_requests", + "process_cpu_time_seconds_total", "process_memory_usage_bytes", "otelcol_receiver_accepted_spans_total", + "otelcol_exporter_sent_spans_total", "otelcol_exporter_queue_size", +] + + +def main(): + c = Client() + alice, bob, _eve, chat = c.bootstrap() + marker = "secret-text-" + uuid.uuid4().hex + sent = c.message(alice, chat, marker) + msg_rid = sent["headers"]["X-Request-Id"] + log("message sent request_id=%s (text marker kept private)" % msg_rid) + + # ---------------------------------------------------------------- 1. upload -> retry -> ready, one trace + filename = unique_name("transient1") + att = upload(c, alice, chat, filename=filename) + ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90) + check(ready["status"] == "ready", "upload must become ready after one retry") + log("upload %s: %s" % (att["id"], " -> ".join(["queued"] + seen))) + + def find(): + found = obs.tempo_search('{span.messenger.attachment_id="%s"}' % att["id"]) + if not found: + return None + spans = obs.tempo_trace(found[0]["traceID"]) + consumers = [s for s in spans if s["kind"] == "CONSUMER"] + producers = [s for s in spans if s["kind"] == "PRODUCER"] + # Spans of one trace arrive from 3 processes with different batch delays: wait until all are in. + return (found, spans) if len(consumers) >= 2 and len(producers) >= 2 and any( + s["attrs"].get("messenger.job.outcome") == "succeeded" for s in consumers) else None + + result = obs.wait_for(find, 60) + check(result, "trace of the upload with both attempts not found in Tempo") + found, spans = result + trace_ids = {t["traceID"] for t in found} + check(len(trace_ids) == 1, "upload and its jobs must be ONE trace, got %s" % trace_ids) + trace_id = found[0]["traceID"] + log("Tempo: trace %s, %d spans; tree (DB client spans omitted):" % (trace_id, len(spans))) + obs.print_tree(spans, out=lambda line: print(" " + line), skip_db=True) + + by_id = {s["span_id"]: s for s in spans} + server = [s for s in spans if s["kind"] == "SERVER" and s["service"] == "messenger-api"] + producers = [s for s in spans if s["kind"] == "PRODUCER"] + consumers = [s for s in spans if s["kind"] == "CONSUMER"] + check(len(server) == 1 and server[0]["name"] == "POST /api/v1/chats/{chat_id}/attachments", "API server span") + check(len(producers) >= 2 and len(consumers) >= 2, "two publish/consume pairs (attempt 1 + retry)") + for cons in consumers: + parent = by_id.get(cons["parent_id"]) + check(parent is not None and parent["kind"] == "PRODUCER", "consumer must be a child of a producer span") + check(by_id[producers[0]["parent_id"]]["kind"] == "SERVER", "first producer is a child of the upload request") + retry_parent = by_id[producers[1]["parent_id"]] + check(retry_parent["kind"] == "CONSUMER" and retry_parent["attrs"].get("messenger.job.outcome") == "retry", + "retry producer must be a child of the failed consumer") + check(consumers[0]["status"] == "ERROR" and consumers[-1]["attrs"].get("messenger.job.outcome") == "succeeded", + "attempt 1 ERROR/retry, last attempt succeeded") + worker_ops = {s["name"] for s in spans if s["service"] == "messenger-worker"} + for op in ("S3.GetObject", "S3.CopyObject", "S3.PutObject", "imageproc"): + check(op in worker_ops, "worker span %s missing" % op) + check(any(s["service"] == "messenger-worker" and s["attrs"].get("db.system") == "postgresql" for s in spans), + "worker DB spans missing") + log("PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links") + + # ---------------------------------------------------------------- 2. logs of both services with the trace_id + lines = obs.wait_for(lambda: [l for l in obs.loki('{service_name=~"messenger-api|messenger-worker"} | trace_id="%s"' + % trace_id) if True] or None, 30) + check(lines, "no Loki records with the trace_id") + services = {l["service_name"] for l in lines} + events = [(l["service_name"].replace("messenger-", ""), l.get("event")) for l in lines] + log("Loki: %d records with trace_id=%s: %s" % (len(lines), trace_id, events)) + check(services == {"messenger-api", "messenger-worker"}, "logs of both services must carry the trace_id") + check({"attachment_queued", "job_attempt_failed", "job_finished"} <= {e for _, e in events}, + "expected events attachment_queued, job_attempt_failed, job_finished") + sample = lines[0] + log("Loki record fields: %s" % sorted(k for k in sample if k not in ("line",))) + + # ---------------------------------------------------------------- 3. X-Request-Id of an HTTP error -> log -> trace + err = c.call("GET", "/api/v1/chats", 500, actor=alice, headers={"X-Fault": "error"}) + rid = err["headers"]["X-Request-Id"] + check(err["json"]["error"]["request_id"] == rid and "Traceback" not in err["bytes"].decode(), "500 body is clean") + err_lines = obs.wait_for(lambda: obs.loki('{service_name="messenger-api"} |= "%s"' % rid, minutes=5) or None, 30) + check(err_lines, "500 request_id not found in Loki") + err_trace = err_lines[0].get("trace_id") + log("HTTP 500 X-Request-Id=%s -> Loki event=%s trace_id=%s" % (rid, err_lines[0].get("event"), err_trace)) + err_spans = obs.wait_for(lambda: obs.tempo_trace(err_trace) or None, 30) + check(err_spans, "trace of the 500 not in Tempo") + srv = [s for s in err_spans if s["kind"] == "SERVER"][0] + check(srv["attrs"].get("messenger.request_id") == rid and srv["status"] == "ERROR" + and str(srv["attrs"].get("http.response.status_code")) == "500", "server span must carry request_id and ERROR") + log("Tempo: %s status=%s request_id matches; exception event recorded=%s" % ( + srv["name"], srv["status"], "exception" in obs.tempo_raw(err_trace))) + + msg_trace = obs.wait_for(lambda: obs.tempo_search('{span.messenger.request_id="%s"}' % msg_rid) or None, 30) + check(msg_trace, "message request trace not found by request_id") + msg_spans = obs.tempo_trace(msg_trace[0]["traceID"]) + log("message POST trace %s: %s" % (msg_trace[0]["traceID"], [s["name"] for s in msg_spans])) + + # ---------------------------------------------------------------- 4. privacy + secrets = {"password": alice["registration"]["password"], "token": alice["token"], "message text": marker, + "filename": filename} + raw = obs.tempo_raw(trace_id) + obs.tempo_raw(msg_trace[0]["traceID"]) + obs.tempo_raw(err_trace) + logs_raw = "\n".join(str(l) for l in obs.loki('{service_name=~"messenger-.*"}', minutes=10, limit=5000)) + for what, value in secrets.items(): + check(value not in raw, "%s leaked into traces" % what) + check(value not in logs_raw, "%s leaked into logs" % what) + # Image bytes: PNG signature/IHDR chunk as raw, escaped or base64 text (sample.png starts with iVBORw0KGgo). + for word in ("Authorization", "Bearer ", "filename=", "X-Amz-Signature", "iVBORw0KGgo", "IHDR", "\\x89PNG", + "\\u0089PNG"): + check(word not in raw and word not in logs_raw, "%r found in telemetry" % word) + log("PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from " + "traces and logs") + + series = obs.prom('{__name__=~"messenger_.*|http_server_.*|db_client_.*|process_.*"}') + labels = set() + for metric, _ in series: + labels |= set(metric) + for name, value in metric.items(): + check(not UUID_RE.search(value), "UUID-like value in metric label %s=%s" % (name, value)) + check(not labels & FORBIDDEN_LABELS, "forbidden metric labels: %s" % (labels & FORBIDDEN_LABELS)) + log("PASS: %d series, label names: %s" % (len(series), sorted(labels))) + routes = sorted({m.get("http_route") for m, _ in series if m.get("http_route")}) + log("http_route values (templates only): %s" % routes) + + # ---------------------------------------------------------------- 5. metrics referenced by dashboard/alerts + time.sleep(12) # one metric export interval + for name in REQUIRED_METRICS: + check(obs.prom(name), "metric %s has no samples" % name) + log("PASS: required metrics present: %s" % ", ".join(REQUIRED_METRICS)) + print("PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_secret_rotation.py b/student-tests/test_secret_rotation.py new file mode 100644 index 0000000..67a84dc --- /dev/null +++ b/student-tests/test_secret_rotation.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Level 4: rotation of one working secret without image rebuild or restart: the DB password of role app_worker. + + 1. background traffic (messages + 1 upload/s) through HTTPS for the whole test + 2. operator: scripts/bao.sh rotate-worker-db -> ALTER ROLE app_worker + new KV version messenger/worker + 3. both workers pick up the new version from OpenBao (refresh <= BAO_REFRESH_SECONDS) and switch the pool + 4. existing app_worker sessions are terminated in PostgreSQL: the pool must reconnect WITH THE NEW password + 5. old password refused, new password accepted (probe from inside the network); uploads keep reaching ready +Passwords are held in memory only for the probes and are never printed. + +python3 student-tests/test_secret_rotation.py (needs the operator OpenBao material: scripts/bao.sh) +""" +import os +import subprocess +import threading +import time + +from openloop import Load +from stand import ROOT, Client, check, compose, log, run, sql, upload, wait_status + +PROBE = ("import os,psycopg\n" + "try:\n" + " psycopg.connect('postgresql://app_worker@postgres:5432/messenger', password=os.environ['PW']," + " sslmode='verify-full', sslrootcert='/run/secrets/ca.crt', connect_timeout=5).close(); print('accepted')\n" + "except psycopg.Error as e:\n print('rejected: ' + str(e).splitlines()[0][-60:])\n") + + +def probe(password): + out = subprocess.run(["docker", "compose", "run", "--rm", "--no-deps", "-e", "PW", "--entrypoint", "python", + "worker", "-c", PROBE], cwd=ROOT, env={**os.environ, "PW": password}, + capture_output=True, text=True).stdout + return [line for line in out.splitlines() if line.startswith(("accepted", "rejected"))][-1] + + +def worker_logs(since): + return compose("logs", "--since", since, "worker") + + +def main(): + c = Client() + alice, _bob, _eve, chat = c.bootstrap() + load = Load("https://localhost:8443", users=2, threads=32) + stop, summary = threading.Event(), {} + thread = threading.Thread(target=lambda: summary.update(load.run(5, 900, 1, stop_event=stop)), daemon=True) + thread.start() + old = (ROOT / "secrets/bootstrap/db_worker_password").read_text().strip() + check(probe(old) == "accepted", "old password must work before the rotation") + before = int(sql("SELECT count(*) FROM pg_stat_activity WHERE usename = 'app_worker'")[0][0]) + log("before: old password accepted, %d app_worker connections open" % before) + + t0 = time.time() + out = subprocess.run(["sh", "scripts/bao.sh", "rotate-worker-db"], cwd=ROOT, capture_output=True, text=True) + check(out.returncode == 0, "rotation script failed: %s" % out.stderr[-200:]) + log(out.stdout.strip().splitlines()[-1]) + new = (ROOT / "secrets/bootstrap/db_worker_password").read_text().strip() + check(new != old, "password must change") + + deadline = time.time() + 60 + while time.time() < deadline: + logs = worker_logs("90s") + if logs.count("db_credentials_rotated") >= 2: + break + time.sleep(3) + check(logs.count("db_credentials_rotated") >= 2, "both workers must switch to the new secret version") + log("both workers loaded the new KV version and switched the pool (%.0f s after rotation)" % (time.time() - t0)) + + sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = 'app_worker'") + log("terminated all existing app_worker sessions: the pools must reconnect with the new password") + atts = [upload(c, alice, chat, filename="rotation-%d.png" % i) for i in range(3)] + for att in atts: + wait_status(c, alice, att["id"], ("ready",), 90) + after = int(sql("SELECT count(*) FROM pg_stat_activity WHERE usename = 'app_worker'")[0][0]) + log("3 uploads after the forced reconnect are ready; %d app_worker connections open again" % after) + check(after > 0, "workers must reconnect") + old_result, new_result = probe(old), probe(new) + log("probe: old password -> %s | new password -> %s" % (old_result, new_result)) + check(old_result.startswith("rejected") and new_result == "accepted", "old refused, new accepted") + failed = sql("SELECT count(*) FROM attachment_jobs WHERE state = 'failed' AND updated_at > to_timestamp(%d)" % t0)[0][0] + stop.set() + thread.join(60) + log("traffic: %s requests, unexpected errors %s, uploads %s; failed jobs since rotation: %s" % ( + summary.get("requests"), summary.get("server_errors_unexpected"), summary["by_kind"]["upload"]["n"], failed)) + check(summary.get("server_errors_unexpected") == 0 and failed == "0", "no errors and no failed jobs") + print("PASS: app_worker password rotated via OpenBao without rebuild/restart; old password revoked") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker_down.py b/student-tests/test_worker_down.py new file mode 100755 index 0000000..49fc71e --- /dev/null +++ b/student-tests/test_worker_down.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Grade 3 demo: worker stopped -> upload 202 -> message with attachment -> restart API + broker (volumes kept) +-> worker started -> ready + thumbnail. Then an explicit duplicate delivery of the finished job is a no-op. + +python3 student-tests/test_worker_down.py (stack from REPORT.md must be running) +""" +import hashlib +import time + +from stand import (Client, check, compose, started_at, get, log, objects_of, wait_queue, run, sql, upload, wait_http_ready, + wait_status, unique_name, SAMPLE) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + log("stop all workers") + compose("stop", "-t", "20", "worker") + try: + att = upload(c, alice, chat, filename=unique_name()) + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload must be 202 queued") + log("upload 202 queued id=%s" % att["id"]) + msg = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + log("message %s created with queued attachment" % msg["id"]) + c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob) + c.message(bob, chat, "другие сообщения работают") + depth = wait_queue() + log("broker queue while worker is down:", depth) + check(depth and depth["ready"] >= 1, "job must be waiting in the broker") + unpublished = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0] + check(unpublished == "0", "outbox row must be published with confirm") + + log("restart api-1 api-2 rabbitmq (volumes kept)") + before = started_at("api-1", "api-2", "rabbitmq") + compose("restart", "-t", "10", "rabbitmq", "api-1", "api-2") + wait_http_ready(c) + after = started_at("api-1", "api-2", "rabbitmq") + log("started before:", before) + log("started after: ", after) + check(before != after, "containers must really restart") + check(get(c, bob, att["id"])["status"] == "queued", "still queued after restart, nothing lost") + depth = wait_queue() + log("broker queue after broker restart:", depth) + check(depth and depth["ready"] >= 1, "durable message must survive broker restart") + finally: + log("start workers") + compose("start", "worker") + ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90) + check(ready["status"] == "ready", "expected ready, got %s" % ready["status"]) + log("status sequence observed:", " -> ".join(seen)) + names = sorted(v["name"] for v in ready["variants"]) + check(names == ["crop", "optimized", "thumbnail"], "expected thumbnail+crop+optimized, got %s" % names) + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail 256x160") + body = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"] + check(hashlib.sha256(body).hexdigest() == thumb["sha256"], "thumbnail checksum") + orig = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"] + check(orig == SAMPLE, "original unchanged byte for byte") + keys = objects_of(att["id"]) + log("objects:", keys) + check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), "original + 3 variants, staging removed") + + log("redelivery: republish the finished job through the outbox (duplicate message)") + sql("INSERT INTO outbox (job_id, payload, available_at, created_at) SELECT o.job_id, o.payload, now(), now()" + " FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='%s' ORDER BY o.id LIMIT 1" + % att["id"]) + time.sleep(5) + again = get(c, bob, att["id"]) + check(again["status"] == "ready" and again["variants"] == ready["variants"], "duplicate must not change the result") + rows = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att["id"])[0][0] + succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att["id"])[0][0] + check(rows == "3" and succeeded == "1", "one set of variants and one success (rows=%s succeeded=%s)" % (rows, succeeded)) + check(len(objects_of(att["id"])) == 4, "no new objects after duplicate") + noop = compose("logs", "--since", "30s", "worker") + check("decision=done" in noop, "worker must log the duplicate as no-op") + log("duplicate acked as no-op: variants=%s successes=%s" % (rows, succeeded)) + print("PASS: worker down / API+broker restart / redelivery no-op") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker_saturation.py b/student-tests/test_worker_saturation.py new file mode 100644 index 0000000..7824b16 --- /dev/null +++ b/student-tests/test_worker_saturation.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""MessengerWorkerSaturated: capacity is lowered (2 workers x 1 slot, 0.25 CPU each), uploads exceed it for longer than +`for: 5m`; the alert must fire, and resolve after the normal configuration is restored and the backlog drains. + +python3 student-tests/test_worker_saturation.py (stack running; ~10 min). Restores WORKER_* defaults at the end. +""" +import json +import os +import subprocess +import threading +import time +from pathlib import Path + +import obs +from openloop import Load +from stand import check, log, run +from test_incidents import wait_alert + +ROOT = Path(__file__).resolve().parents[1] +ALERT = "MessengerWorkerSaturated" + + +def worker(env): + done = subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--no-deps", "--force-recreate", + "worker"], cwd=ROOT, env={**os.environ, **env}, capture_output=True, text=True, timeout=240) + check(done.returncode == 0, "worker recreate failed: %s" % done.stderr[-300:]) + + +def main(): + result = {} + log("lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas") + worker({"WORKER_CONCURRENCY": "1", "WORKER_PREFETCH": "1", "WORKER_CPUS": "0.25"}) + load = Load("https://localhost:8443", users=2, threads=32) + stop = threading.Event() + summary = {} + thread = threading.Thread(target=lambda: summary.update(load.run(0, 900, upload_rate=4, stop_event=stop)), daemon=True) + t0 = time.time() + thread.start() + log("uploads: 4/s open-loop (sample.png)") + try: + time.sleep(60) + capacity = obs.prom_scalar("sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))") + log("fresh capacity=%.0f slots (stale series of recreated containers are excluded)" % capacity) + check(capacity == 2, "expected 2 slots, got %s" % capacity) + firing, pending = wait_alert(ALERT, "firing", 480, t0) + ratio = obs.prom_scalar("messenger:worker_saturation:ratio") + depth = obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})') + age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})') + rate = obs.prom_scalar('sum(rate(messenger_jobs_processed_total{outcome="succeeded"}[2m]))') + p95 = obs.prom_scalar('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[5m])))') + others = obs.alerts() + log("at firing: saturation=%.2f, queued=%.0f, oldest age=%.0f s, succeeded=%.2f jobs/s, job p95=%.2f s, alerts=%s" + % (ratio, depth, age, rate, p95, others)) + check(ratio > 0.9, "saturation must be above the threshold") + finally: + stop.set() + thread.join(60) + log("restore default worker configuration") + t1 = time.time() + worker({}) + resolved, _ = wait_alert(ALERT, "resolved", 240, t1) + drained = obs.wait_for(lambda: obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})', + default=-1) == 0, 240, 5) + log("backlog drained: %s (+%.0f s after restore)" % (bool(drained), time.time() - t1)) + check(drained, "backlog must drain") + result.update({"alert_pending_s": pending, "alert_firing_s": round(firing), "saturation_at_firing": round(ratio, 3), + "queued_at_firing": depth, "oldest_age_s": round(age), "jobs_per_s": round(rate, 2), + "job_p95_s": round(p95, 2), "alerts_at_firing": others, "resolved_after_restore_s": round(resolved), + "drained_after_restore_s": round(time.time() - t1), "uploads": summary.get("status_counts")}) + (ROOT / "evidence/worker_saturation.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print("PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/tls_audit.py b/student-tests/tls_audit.py new file mode 100644 index 0000000..07bd99b --- /dev/null +++ b/student-tests/tls_audit.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +"""TLS / mTLS / authorization audit of the whole trust matrix (contracts/trust-matrix.md), from INSIDE the network. + +Run (stand up, OpenBao unsealed for section H): + docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro \ + -v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py [--only A,B,...] +For every link: no client certificate / certificate of a FOREIGN CA (claims the right name) / certificate of the +SAME CA with a wrong identity / the right identity. Stores additionally get allow/deny checks of the scoped rights. +Credentials from /bootstrap are used only for positive probes and are never printed. Exit 1 on any mismatch. +""" +import argparse +import asyncio +import json +import socket +import ssl +import sys +import time +import urllib.error +import urllib.request + +PKI = "/pki" +CA = PKI + "/ca.crt" +FOREIGN_CA = PKI + "/foreign.crt" +RESULTS = [] + + +def cert(name): + return (PKI + "/%s.crt" % name, PKI + "/%s.key" % name) if name else None + + +def secret(name): + with open("/bootstrap/" + name, encoding="utf-8") as handle: + return handle.read().strip() + + +def record(section, link, case, expect, observed, ok): + RESULTS.append({"section": section, "link": link, "case": case, "expect": expect, "observed": observed[:160], + "pass": bool(ok)}) + print("%s %-2s %-26s %-44s expect=%-8s %s" % ("PASS" if ok else "FAIL", section, link, case, expect, observed[:110]), + flush=True) + + +def ctx_for(client=None, cafile=CA, alpn=None): + ctx = ssl.create_default_context(cafile=cafile) + if client: + ctx.load_cert_chain(*cert(client)) + if alpn: + ctx.set_alpn_protocols(alpn) + return ctx + + +def tls_exchange(host, port, client=None, cafile=CA, server_hostname=None, payload=b"", timeout=3.0, alpn=None): + """TLS connect + optional payload + read. Returns ('accepted', first bytes) or ('rejected', reason). + With TLS 1.3 a missing/untrusted client certificate is reported by the server AFTER the client's handshake, + so the verdict needs a read; a closed connection without data counts as rejected.""" + raw = socket.create_connection((host, port), timeout=timeout) + try: + s = ctx_for(client, cafile, alpn).wrap_socket(raw, server_hostname=server_hostname or host) + except ssl.SSLCertVerificationError as exc: + raw.close() + return "rejected", "server cert verify failed: " + exc.verify_message + except (ssl.SSLError, OSError) as exc: + raw.close() + return "rejected", "handshake: " + (getattr(exc, "reason", None) or type(exc).__name__) + try: + if payload: + s.sendall(payload) + data = s.recv(64) + return ("accepted", data.decode("latin-1", "replace")) if data else ("rejected", "closed without response") + except socket.timeout: + return "accepted", "(open, no data within %.1fs)" % timeout + except (ssl.SSLError, OSError) as exc: + return "rejected", "after handshake: " + (getattr(exc, "reason", None) or type(exc).__name__) + finally: + s.close() + + +def http_get(host, port, path, client=None, hostname=None): + payload = ("GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n" % (path, hostname or host)).encode() + verdict, detail = tls_exchange(host, port, client, server_hostname=hostname, payload=payload) + if verdict == "accepted" and detail.startswith("HTTP/"): + return "accepted", detail.split("\r\n")[0] + return verdict, detail + + +def expect_tls(section, link, host, port, client, case, accept, path=None, payload=b"", hostname=None, alpn=None): + if path: + verdict, detail = http_get(host, port, path, client, hostname) + else: + verdict, detail = tls_exchange(host, port, client, payload=payload, server_hostname=hostname, alpn=alpn) + record(section, link, case, "accept" if accept else "reject", "%s: %s" % (verdict, detail), + (verdict == "accepted") == accept) + + +def plaintext_refused(section, link, host, port, probe=b"GET / HTTP/1.0\r\n\r\n"): + try: + s = socket.create_connection((host, port), timeout=3) + s.sendall(probe) + data = s.recv(64) + s.close() + ok = not data.startswith((b"HTTP/1.1 200", b"AMQP", b"+PONG", b"-NOAUTH")) + record(section, link, "plaintext client", "reject", "server answered %r" % data[:24], ok) + except OSError as exc: + record(section, link, "plaintext client", "reject", "connection failed: %s" % type(exc).__name__, True) + + +# ---------------------------------------------------------------------------------------------- sections +def a_lb_api(): + for api in ("api-1", "api-2"): + link = "LB -> %s:8080" % api + expect_tls("A", link, api, 8080, None, "no client cert", False, "/health/ready") + expect_tls("A", link, api, 8080, "foreign-client", "foreign CA (claims lb identity)", False, "/health/ready") + expect_tls("A", link, api, 8080, "intruder", "same CA, identity intruder", False, "/health/ready") + expect_tls("A", link, api, 8080, "worker", "same CA, identity worker", False, "/health/ready") + expect_tls("A", link, api, 8080, "api", "same CA, identity api", False, "/health/ready") + expect_tls("A", link, api, 8080, "lb-client", "lb.client.messenger", True, "/health/ready") + plaintext_refused("A", link, api, 8080) + + +def i_ingress(): + link = "client -> nginx:8443" + expect_tls("I", link, "nginx", 8443, None, "trusted CA, hostname nginx", True, "/health/live") + verdict, detail = tls_exchange("nginx", 8443, cafile=FOREIGN_CA) + record("I", link, "untrusted CA in client trust store", "reject", "%s: %s" % (verdict, detail), + verdict == "rejected" and "verify failed" in detail) + verdict, detail = tls_exchange("nginx", 8443, server_hostname="evil.example") + record("I", link, "wrong hostname (evil.example)", "reject", "%s: %s" % (verdict, detail), + verdict == "rejected" and "verify failed" in detail) + plaintext_refused("I", link, "nginx", 8443) + + +def b_postgres(): + import psycopg + + link = "svc -> postgres:5432" + url = lambda role, pw: "postgresql://%s:%s@postgres:5432/messenger" % (role, pw) # noqa: E731 + api_pw, worker_pw = secret("db_api_password"), secret("db_worker_password") + + def connect(role, pw, **kw): + return psycopg.connect(url(role, pw), connect_timeout=5, autocommit=True, **kw) + + def attempt(case, accept, **kw): + try: + connect("app_api", api_pw, **kw).close() + record("B", link, case, "accept" if accept else "reject", "connected", accept) + except psycopg.Error as exc: + record("B", link, case, "accept" if accept else "reject", type(exc).__name__ + ": " + str(exc).splitlines()[0], + not accept) + + attempt("plaintext (sslmode=disable)", False, sslmode="disable") + attempt("foreign CA as root (verify-full)", False, sslmode="verify-full", sslrootcert=FOREIGN_CA) + ip = socket.gethostbyname("postgres") + try: + psycopg.connect("postgresql://app_api:%s@%s:5432/messenger" % (api_pw, ip), sslmode="verify-full", + sslrootcert=CA, connect_timeout=5).close() + record("B", link, "hostname mismatch (IP, verify-full)", "reject", "connected", False) + except psycopg.Error as exc: + record("B", link, "hostname mismatch (IP, verify-full)", "reject", str(exc).splitlines()[0], True) + attempt("TLS verify-full + app_api password", True, sslmode="verify-full", sslrootcert=CA) + try: + connect("app_api", "wrong-password", sslmode="verify-full", sslrootcert=CA).close() + record("B", link, "wrong password", "reject", "connected", False) + except psycopg.Error as exc: + record("B", link, "wrong password", "reject", str(exc).splitlines()[0], True) + + checks = [ + ("app_worker", worker_pw, "SELECT count(*) FROM users", False), + ("app_worker", worker_pw, "SELECT count(*) FROM messages", False), + ("app_worker", worker_pw, "SELECT count(*) FROM sessions", False), + ("app_worker", worker_pw, "DELETE FROM attachments WHERE false", False), + ("app_worker", worker_pw, "SELECT count(*) FROM attachment_jobs", True), + ("app_api", api_pw, "SELECT count(*) FROM users", True), + ("app_api", api_pw, "CREATE TABLE audit_probe (x int)", False), + ("app_api", api_pw, "ALTER ROLE app_worker PASSWORD 'x'", False), + ("app_api", api_pw, "SELECT count(*) FROM schema_migrations", False), + ] + for role, pw, query, accept in checks: + try: + with connect(role, pw, sslmode="verify-full", sslrootcert=CA) as conn: + conn.execute(query) + record("B", "%s privileges" % role, query[:44], "allow" if accept else "deny", "executed", accept) + except psycopg.Error as exc: + record("B", "%s privileges" % role, query[:44], "allow" if accept else "deny", + type(exc).__name__, not accept) + + +def c_s3(): + import boto3 + from botocore.config import Config + from botocore.exceptions import ClientError, SSLError, EndpointConnectionError + + bucket = "messenger-attachments" + + def client(role, verify=CA, endpoint="https://minio:9000"): + return boto3.client("s3", endpoint_url=endpoint, region_name="us-east-1", verify=verify, + aws_access_key_id=secret("s3_%s_access_key" % role), + aws_secret_access_key=secret("s3_%s_secret_key" % role), + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, + retries={"max_attempts": 0}, connect_timeout=3, read_timeout=5)) + + def op(role, case, accept, fn): + try: + fn(client(role)) + record("C", "%s S3 rights" % role, case, "allow" if accept else "deny", "ok", accept) + except ClientError as exc: + record("C", "%s S3 rights" % role, case, "allow" if accept else "deny", exc.response["Error"]["Code"], + not accept) + + tag = "audit-%d" % int(time.time()) + try: + client("api", verify=FOREIGN_CA).list_objects_v2(Bucket=bucket, MaxKeys=1) + record("C", "svc -> minio:9000", "foreign CA as trust anchor", "reject", "connected", False) + except (SSLError, EndpointConnectionError, Exception) as exc: # noqa: B014 + record("C", "svc -> minio:9000", "foreign CA as trust anchor", "reject", type(exc).__name__, True) + try: + client("api", endpoint="http://minio:9000").list_objects_v2(Bucket=bucket, MaxKeys=1) + record("C", "svc -> minio:9000", "plaintext http://", "reject", "answered", False) + except Exception as exc: # noqa: BLE001 + record("C", "svc -> minio:9000", "plaintext http://", "reject", type(exc).__name__, True) + op("worker", "PUT attachments/*", True, lambda c: c.put_object(Bucket=bucket, Key="attachments/%s/x" % tag, Body=b"x")) + op("worker", "GET attachments/*", False, lambda c: c.get_object(Bucket=bucket, Key="attachments/%s/x" % tag)) + op("worker", "DELETE attachments/*", False, lambda c: c.delete_object(Bucket=bucket, Key="attachments/%s/x" % tag)) + op("worker", "PUT staging/*", False, lambda c: c.put_object(Bucket=bucket, Key="staging/%s" % tag, Body=b"x")) + op("worker", "LIST bucket", False, lambda c: c.list_objects_v2(Bucket=bucket, MaxKeys=1)) + op("worker", "PUT outside prefixes", False, lambda c: c.put_object(Bucket=bucket, Key="other/%s" % tag, Body=b"x")) + op("api", "PUT staging/*", True, lambda c: c.put_object(Bucket=bucket, Key="staging/%s" % tag, Body=b"x")) + op("api", "PUT attachments/* (only worker publishes)", False, + lambda c: c.put_object(Bucket=bucket, Key="attachments/%s/y" % tag, Body=b"x")) + op("api", "GET attachments/*", True, lambda c: c.get_object(Bucket=bucket, Key="attachments/%s/x" % tag)) + op("api", "LIST bucket (reconciler)", True, lambda c: c.list_objects_v2(Bucket=bucket, MaxKeys=1)) + op("api", "create bucket", False, lambda c: c.create_bucket(Bucket="audit-%s" % tag)) + op("api", "cleanup audit objects", True, lambda c: (c.delete_object(Bucket=bucket, Key="staging/%s" % tag), + c.delete_object(Bucket=bucket, Key="attachments/%s/x" % tag))) + + +def d_broker(): + import aio_pika + from aiormq.exceptions import AMQPError + + link = "svc -> rabbitmq:5671" + url = "amqps://rabbitmq:5671/messenger?auth=external" + plaintext_refused("D", "svc -> rabbitmq:5672", "rabbitmq", 5672, b"AMQP\x00\x00\x09\x01") + + async def connect(client): + ctx = ssl.create_default_context(cafile=CA) + if client: + ctx.load_cert_chain(*cert(client)) + return await aio_pika.connect(url, ssl_context=ctx, timeout=5) + + async def run(): + for client, case in ((None, "no client cert"), ("foreign-client", "foreign CA (claims worker name)"), + ("intruder", "same CA, unknown identity intruder"), ("lb-client", "same CA, identity lb")): + try: + conn = await connect(client) + await conn.close() + record("D", link, case, "reject", "connected", False) + except (AMQPError, ConnectionError, OSError, ssl.SSLError, asyncio.TimeoutError) as exc: + record("D", link, case, "reject", type(exc).__name__ + " " + str(exc)[:60], True) + + async def action(client, case, accept, fn): + conn = None + try: + conn = await connect(client) + channel = await conn.channel(publisher_confirms=True) + await asyncio.wait_for(fn(channel), 5) + record("D", "%s AMQP rights" % client, case, "allow" if accept else "deny", "ok", accept) + except (AMQPError, asyncio.TimeoutError, aio_pika.exceptions.ChannelClosed) as exc: + record("D", "%s AMQP rights" % client, case, "allow" if accept else "deny", + type(exc).__name__ + " " + str(exc)[:70], not accept) + finally: + if conn is not None: + await conn.close() + + async def publish_unroutable(channel): + ex = await channel.get_exchange("attachments", ensure=True) + await ex.publish(aio_pika.Message(b"audit"), routing_key="audit.unroutable") + + async def consume_and_cancel(channel): + q = await channel.get_queue("attachments.process", ensure=True) + await channel.set_qos(prefetch_count=1) + tag = await q.consume(lambda m: m.nack(requeue=True)) + await q.cancel(tag) + + async def declare_queue(channel): + await channel.declare_queue("audit-probe", auto_delete=True) + + async def publish_default(channel): + await channel.default_exchange.publish(aio_pika.Message(b"audit"), routing_key="attachments.process") + + await action("api", "publish to exchange attachments", True, publish_unroutable) + await action("api", "consume attachments.process", False, consume_and_cancel) + await action("api", "declare a queue (configure)", False, declare_queue) + await action("api", "publish via default exchange", False, publish_default) + await action("worker", "consume attachments.process", True, consume_and_cancel) + await action("worker", "publish to exchange attachments", False, publish_unroutable) + await action("worker", "declare a queue (configure)", False, declare_queue) + + asyncio.run(run()) + + +def e_collector(): + link = "svc -> otel-collector:4318" + body = b"POST /v1/metrics HTTP/1.1\r\nHost: otel-collector\r\nContent-Type: application/x-protobuf\r\n" \ + b"Content-Length: 0\r\nConnection: close\r\n\r\n" + for client, case, accept in ((None, "no client cert", False), ("foreign-client", "foreign CA", False), + ("lb-client", "root CA identity lb (not ingest CA)", False), + ("api", "root CA identity api (not ingest CA)", False), + ("grafana-out", "telemetry CA (not ingest CA)", False), + ("api-otel", "ingest CA api.otel", True), ("worker-otel", "ingest CA worker.otel", True)): + verdict, detail = tls_exchange("otel-collector", 4318, client, payload=body) + ok = (verdict == "accepted" and detail.startswith("HTTP/1.1 2")) if accept else verdict == "rejected" + record("E", link, case, "accept" if accept else "reject", "%s: %s" % (verdict, detail.split("\r\n")[0]), ok) + + +def f_backends(): + for host, port, path in (("prometheus", 9090, "/-/ready"), ("loki", 3100, "/ready"), ("tempo", 3200, "/ready")): + link = "-> %s:%d" % (host, port) + for client, case, accept in ((None, "no client cert", False), ("foreign-client", "foreign CA", False), + ("api", "root CA identity api", False), + ("api-otel", "ingest CA (apps cannot bypass collector)", False), + ("collector-out", "telemetry CA collector.out", True), + ("grafana-out", "telemetry CA grafana.out", True), + ("operator", "telemetry CA operator", True)): + expect_tls("F", link, host, port, client, case, accept, path) + link = "collector -> tempo:4317 (gRPC)" + preface = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + for client, case, accept in ((None, "no client cert", False), ("api-otel", "ingest CA", False), + ("collector-out", "telemetry CA collector.out", True)): + expect_tls("F", link, "tempo", 4317, client, case, accept, payload=preface, alpn=["h2"]) + + +def g_valkey(): + import valkey + + link = "api -> valkey:6379" + plaintext_refused("G", link, "valkey", 6379, b"PING\r\n") + + def conn(user=None, password=None, cafile=CA): + return valkey.Valkey(host="valkey", port=6379, ssl=True, ssl_ca_certs=cafile, ssl_check_hostname=True, + username=user, password=password, socket_timeout=3, decode_responses=True) + + def attempt(case, accept, fn): + try: + fn() + record("G", link, case, "allow" if accept else "deny", "ok", accept) + except Exception as exc: # noqa: BLE001 + record("G", link, case, "allow" if accept else "deny", type(exc).__name__ + " " + str(exc)[:60], not accept) + + attempt("foreign CA as trust anchor", False, lambda: conn("api", secret("valkey_api_password"), FOREIGN_CA).ping()) + attempt("no authentication (default user off)", False, lambda: conn().get("session:x")) + attempt("user api, wrong password", False, lambda: conn("api", "wrong").ping()) + api = lambda: conn("api", secret("valkey_api_password")) # noqa: E731 + attempt("api: SET/GET session:*", True, lambda: (api().set("session:audit", "1", ex=5), api().get("session:audit"))) + attempt("api: INCR ratelimit:*", True, lambda: api().incr("ratelimit:audit")) + attempt("api: GET outside allowed keys", False, lambda: api().get("other:audit")) + attempt("api: FLUSHALL", False, lambda: api().flushall()) + attempt("api: CONFIG GET", False, lambda: api().config_get("*")) + attempt("api: KEYS *", False, lambda: api().keys("*")) + attempt("health: GET session:*", False, lambda: conn("health", "health").get("session:audit")) + + +def h_openbao(): + base = "https://openbao:8200" + + def call(path, client=None, token=None, body=None, method=None): + req = urllib.request.Request(base + path, data=json.dumps(body).encode() if body is not None else None, + method=method or ("POST" if body is not None else "GET")) + if token: + req.add_header("X-Vault-Token", token) + try: + with urllib.request.urlopen(req, context=ctx_for(client), timeout=5) as response: + return response.status, json.loads(response.read() or b"{}") + except urllib.error.HTTPError as exc: + return exc.code, {} + except (urllib.error.URLError, OSError) as exc: + return 0, {"error": type(getattr(exc, "reason", exc)).__name__} + + def login(client, role): + status, body = call("/v1/auth/cert/login", client, body={"name": role}) + return status, (body.get("auth") or {}).get("client_token") + + link = "svc -> openbao:8200" + for client, role, case in ((None, "worker", "no client cert"), ("intruder", "worker", "same CA, identity intruder"), + ("worker", "api", "worker cert, role api"), ("api", "worker", "api cert, role worker"), + ("lb-client", "api", "lb cert, role api")): + status, token = login(client, role) + record("H", link, "cert login: " + case, "reject", "HTTP %s token=%s" % (status, bool(token)), not token) + # /v1/sys/health is public by design; what matters is that a foreign certificate never becomes an identity. + status, token = login("foreign-client", "worker") + record("H", link, "cert login: foreign CA (claims worker name)", "reject", "HTTP %s token=%s" % (status, bool(token)), + not token) + for client, role, own, other in (("worker", "worker", "worker", "api"), ("api", "api", "api", "worker")): + status, token = login(client, role) + record("H", link, "cert login: %s cert, role %s" % (client, role), "accept", "HTTP %s token=%s" % (status, bool(token)), + bool(token)) + if not token: + continue + lookup = call("/v1/auth/token/lookup-self", token=token)[1].get("data", {}) + record("H", "%s token" % role, "policies / ttl", "scoped", "policies=%s ttl=%ss" % ( + lookup.get("policies"), lookup.get("ttl")), lookup.get("policies") == ["default", "messenger-" + role] + and 0 < lookup.get("ttl", 0) <= 900) + status, _ = call("/v1/messenger/data/" + own, token=token) + record("H", "%s token" % role, "read own messenger/%s" % own, "allow", "HTTP %s" % status, status == 200) + status, _ = call("/v1/messenger/data/" + other, token=token) + record("H", "%s token" % role, "read messenger/%s" % other, "deny", "HTTP %s" % status, status == 403) + status, _ = call("/v1/messenger/data/" + own, token=token, body={"data": {"x": "y"}}) + record("H", "%s token" % role, "write own secret", "deny", "HTTP %s" % status, status == 403) + status, _ = call("/v1/sys/policies/acl/messenger-" + role, token=token, body={"policy": 'path "*" {capabilities=["sudo"]}'}, + method="PUT") + record("H", "%s token" % role, "write own policy", "deny", "HTTP %s" % status, status == 403) + status, _ = call("/v1/sys/auth", token=token) + record("H", "%s token" % role, "list auth methods (admin)", "deny", "HTTP %s" % status, status == 403) + + +def j_operator(): + link = "operator -> grafana:3000" + verdict, detail = http_get("grafana", 3000, "/api/datasources") + record("J", link, "HTTPS without login", "reject", "%s: %s" % (verdict, detail), "401" in detail) + plaintext_refused("J", link, "grafana", 3000) + + +def k_bypass(): + """Side channels must not be reachable from the network (loopback-only or not listening at all).""" + closed = [("rabbitmq", 25672, "Erlang distribution"), ("rabbitmq", 4369, "epmd"), ("rabbitmq", 15672, "management UI"), + ("loki", 9096, "Loki internal gRPC"), ("tempo", 9095, "Tempo internal gRPC"), + ("otel-collector", 8888, "collector self-metrics"), ("otel-collector", 13133, "collector health"), + ("otel-collector", 4317, "collector OTLP/gRPC (disabled)"), ("api-1", 8081, "API loopback health"), + ("api-2", 8081, "API loopback health"), ("openbao", 8201, "OpenBao cluster port"), + ("minio", 9001, "MinIO console"), ("postgres", 5433, "no second PostgreSQL listener")] + for host, port, what in closed: + try: + socket.create_connection((host, port), timeout=2).close() + record("K", "%s:%d" % (host, port), what, "closed", "port OPEN from the network", False) + except OSError as exc: + record("K", "%s:%d" % (host, port), what, "closed", type(exc).__name__, True) + + +SECTIONS = {"A": a_lb_api, "I": i_ingress, "B": b_postgres, "C": c_s3, "D": d_broker, "E": e_collector, + "F": f_backends, "G": g_valkey, "H": h_openbao, "J": j_operator, "K": k_bypass} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--only", default=",".join(SECTIONS)) + p.add_argument("--output", default="") + args = p.parse_args() + for key in args.only.split(","): + try: + SECTIONS[key]() + except Exception as exc: # noqa: BLE001 a section that cannot run is a failure, not a skip + record(key, "section " + key, "runs", "run", "%s: %s" % (type(exc).__name__, str(exc)[:120]), False) + failed = [r for r in RESULTS if not r["pass"]] + print("\n%d checks, %d failed" % (len(RESULTS), len(failed))) + if args.output: + with open(args.output, "w", encoding="utf-8") as handle: + json.dump(RESULTS, handle, ensure_ascii=False, indent=1) + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/student-tests/track1.py b/student-tests/track1.py new file mode 100644 index 0000000..ab2ebd6 --- /dev/null +++ b/student-tests/track1.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Track 5.1 "Цена наблюдаемости": the same open-loop load against several telemetry configurations. + +Configurations (applied by recreating api-1, api-2, worker and otel-collector with shell env overrides): + off OTEL_SDK_DISABLED=true no SDK, no instrumentation (baseline) + on100 sampler ratio 1.0, collector profile default every span exported + head10 sampler ratio 0.1, collector profile default head sampling of root spans (children follow) + tail sampler ratio 1.0, collector profile tail balanced: errors + >500 ms + async pipeline + 10% rest + +Per run: recreate -> ready -> warm-up 20 s -> idle 20 s -> counters A -> load (rate, duration) -> idle 30 s -> +counters B. Delta(B-A) = telemetry produced by the measured load. CPU is the exact cgroup delta over the load window +(Docker API); RSS is the maximum of `docker stats` samples taken every 5 s. Afterwards Tempo is searched in the load window for traces of injected slow/error requests and +upload pipelines, to count what survives sampling. + + python3 student-tests/track1.py run [--configs off,on100,head10,tail] [--repeats 2] [--rate 60] [--duration 90] + python3 student-tests/track1.py summary # evidence/track1/results.jsonl -> evidence/track1/summary.md +""" +import argparse +import json +import os +import re +import statistics +import subprocess +import threading +import time +from pathlib import Path + +import obs +from openloop import Load +from stand import Client, log + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence/track1" +CONFIGS = { + "off": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, + "on100": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, + "head10": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, + "tail": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, +} +SERVICES = ("api-1", "api-2", "worker", "otel-collector", "tempo", "loki", "prometheus", "postgres", "nginx") +COUNTERS = { + # SDK side (sum over all instances; absent when the SDK is off) + "sdk_span_decisions_sample": 'sum(messenger_telemetry_sampler_decisions_total{decision="sample"})', + "sdk_span_decisions_drop_ratio": 'sum(messenger_telemetry_sampler_decisions_total{decision="drop",rule=~"ratio|parent"})', + "sdk_spans_exported_ok": 'sum(messenger_telemetry_spans_exported_total{outcome="success"})', + "sdk_spans_exported_failed": 'sum(messenger_telemetry_spans_exported_total{outcome="failure"})', + # Collector + "col_spans_in": "sum(otelcol_receiver_accepted_spans_total)", + "col_logs_in": "sum(otelcol_receiver_accepted_log_records_total)", + "col_points_in": "sum(otelcol_receiver_accepted_metric_points_total)", + "col_spans_out": 'sum(otelcol_exporter_sent_spans_total{exporter="otlp/tempo"})', + "col_spans_refused": "sum(otelcol_receiver_refused_spans_total)", + "col_spans_dropped": 'sum(otelcol_exporter_enqueue_failed_spans_total) or vector(0)', + "col_otlp_bytes_in": 'sum({__name__="http.server.request.size_bytes_total"})', + # Backends + "tempo_spans_in": "sum(tempo_distributor_spans_received_total)", + "tempo_bytes_in": "sum(tempo_distributor_bytes_received_total)", + "loki_bytes_in": "sum(loki_distributor_bytes_received_total)", + "loki_lines_in": "sum(loki_distributor_lines_received_total)", + "prom_samples_in": "sum(prometheus_tsdb_head_samples_appended_total)", +} + + +def compose(*args, env=None, timeout=300): + done = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, + text=True, timeout=timeout, env={**os.environ, **(env or {})}) + if done.returncode != 0: + raise RuntimeError("compose %s: %s" % (args[:2], done.stderr[-400:])) + return done.stdout + + +def apply(name): + env = CONFIGS[name] + compose("up", "-d", "--wait", "--force-recreate", "--no-deps", "otel-collector", "api-1", "api-2", "worker", + env=env) + compose("restart", "nginx") # re-resolve recreated replicas + c = Client() + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + break + except Exception: + time.sleep(1) + out = compose("exec", "-T", "api-1", "printenv", "OTEL_SDK_DISABLED", "OTEL_TRACES_SAMPLER_ARG") + log("config %s applied: api-1 env %s" % (name, out.split())) + + +def counters(): + values = {} + for key, expr in COUNTERS.items(): + try: + rows = obs.prom(expr) + values[key] = rows[0][1] if rows else None + except Exception: + values[key] = None + tail = {} + for metric, value in obs.prom('{__name__=~"otelcol_processor_tail_sampling_count_traces_sampled.*"}'): + tail["%s/%s" % (metric.get("policy"), metric.get("sampled"))] = value + values["tail_policies"] = tail + return values + + +def delta(a, b): + out = {} + for key in COUNTERS: + if a.get(key) is not None and b.get(key) is not None: + out[key] = b[key] - a[key] + elif b.get(key) is not None: + out[key] = b[key] # series appeared during the window (counter started at 0) + else: + out[key] = None + out["tail_policies"] = {k: v - a["tail_policies"].get(k, 0) for k, v in b["tail_policies"].items()} + return out + + +def parse_mem(text): + """'123.4MiB' -> 123.4 (MiB).""" + match = re.match(r"\s*([\d.]+)\s*([A-Za-z]*)", text) + if not match: + return 0.0 + scale = {"B": 1 / 1048576, "kB": 1 / 1024, "KiB": 1 / 1024, "MiB": 1, "MB": 1, "GiB": 1024, "GB": 1024} + return float(match.group(1)) * scale.get(match.group(2), 1) + + +def sample_stats(stop, samples): + while not stop.is_set(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], + capture_output=True, text=True).stdout + row = {} + for line in out.splitlines(): + name, cpu, mem = line.split("\t") + if not name.startswith("mtusi-lab6-"): + continue + svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0] + cur = row.setdefault(svc, [0.0, 0.0]) + cur[0] += float(cpu.rstrip("%") or 0) + cur[1] += parse_mem(mem.split(" / ")[0]) + samples.append(row) + stop.wait(5) + + +def docker_socket(): + host = subprocess.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + capture_output=True, text=True).stdout.strip() + return host.replace("unix://", "") + + +def cpu_seconds(): + """Cumulative container CPU seconds per service from the Docker API (exact over a window, unlike docker stats).""" + sock = docker_socket() + ids = subprocess.run(["docker", "ps", "--filter", "label=com.docker.compose.project=mtusi-lab6", "--format", + "{{.ID}} {{.Names}}"], capture_output=True, text=True).stdout.splitlines() + out = {} + for line in ids: + cid, name = line.split() + svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0] + raw = subprocess.run(["curl", "-s", "--unix-socket", sock, + "http://localhost/containers/%s/stats?stream=false&one-shot=true" % cid], + capture_output=True, text=True).stdout + out[svc] = out.get(svc, 0.0) + json.loads(raw)["cpu_stats"]["cpu_usage"]["total_usage"] / 1e9 + return out + + +def count_traces(query, start, end): + try: + return len(obs.tempo_search(query, limit=10000, start=start, end=end)) + except Exception as exc: + return "error:%s" % type(exc).__name__ + + +def one_run(name, rep, rate, duration): + apply(name) + load = Load("https://localhost:8443", users=4, threads=96) + load.run(rate, 20, 0.5, 1, 1) # warm-up, same mix + time.sleep(20) + a = counters() + load = Load("https://localhost:8443", users=4, threads=96) + stop, samples = threading.Event(), [] + sampler = threading.Thread(target=sample_stats, args=(stop, samples), daemon=True) + sampler.start() + cpu_a = cpu_seconds() + t_start = time.time() + summary = load.run(rate, duration, 1, 1, 1) + t_end = time.time() + cpu_b = cpu_seconds() + stop.set() + sampler.join(15) + time.sleep(30) # SDK batch (2 s) + collector batch (2 s) + tail decision_wait (10 s) + scrape (15 s) + b = counters() + d = delta(a, b) + # % of one core averaged over the load window (cumulative cgroup CPU; several replicas of a service are summed) + cpu = {svc: round((cpu_b[svc] - cpu_a[svc]) / (t_end - t_start) * 100, 2) for svc in cpu_b if svc in cpu_a} + mem = {svc: round(max(s[svc][1] for s in samples if svc in s), 1) for svc in samples[0]} if samples else {} + w0, w1 = int(t_start) - 2, int(t_end) + 15 + traces = { + "server_api": count_traces('{resource.service.name="messenger-api" && kind=server}', w0, w1), + "slow": count_traces('{span.messenger.fault="slow"}', w0, w1), + "error": count_traces('{span.messenger.fault="error"}', w0, w1), + "pipeline_consumer": count_traces('{kind=consumer && span.messenger.job.outcome="succeeded"}', w0, w1), + } + result = {"config": name, "rep": rep, "env": CONFIGS[name], "load": summary, "cpu_pct_avg": cpu, + "mem_mib_max": mem, "telemetry_delta": d, "traces_in_tempo": traces, + "window": [t_start, t_end], "stats_samples": len(samples)} + log("%s#%d: ok_rps=%s p50/p95/p99=%s/%s/%s ms api_cpu=%s+%s%% spans_in=%s tempo_spans=%s slow=%s/%d err=%s/%d" % ( + name, rep, summary["successful_rps"], summary["latency_ms_plain_ok"]["p50"], + summary["latency_ms_plain_ok"]["p95"], summary["latency_ms_plain_ok"]["p99"], cpu.get("api-1"), + cpu.get("api-2"), d.get("col_spans_in"), d.get("tempo_spans_in"), traces["slow"], + summary["faults_sent"]["slow"], traces["error"], summary["faults_sent"]["error"])) + return result + + +def cmd_run(args): + OUT.mkdir(parents=True, exist_ok=True) + path = OUT / "results.jsonl" + names = args.configs.split(",") + try: + for rep in range(1, args.repeats + 1): + for name in names: + result = one_run(name, rep, args.rate, args.duration) + result["hardware"] = HARDWARE + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(result, ensure_ascii=False) + "\n") + finally: + log("restore default configuration (on100)") + apply("on100") + + +def fmt(v, digits=0): + if v is None: + return "—" + if isinstance(v, str): + return v + return ("%%.%df" % digits) % v + + +def metric_500s(run_result): + """Injected 500s as seen by METRICS for this run: metrics are computed in the SDK, before any trace sampling.""" + start, end = run_result["window"] + if run_result["config"] == "off": + return None + # Exact counter difference at the window borders (increase() extrapolates and overstates by ~10% here). + expr = 'sum(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code="500"})' + try: + return obs.prom_scalar(expr, at=end + 20) - obs.prom_scalar(expr, at=start - 5) + except Exception: + return None + + +def cmd_summary(_args): + rows = [json.loads(line) for line in (OUT / "results.jsonl").read_text(encoding="utf-8").splitlines() if line] + by = {} + for r in rows: + by.setdefault(r["config"], []).append(r) + + def values(items, fn): + return [v for v in (fn(x) for x in items) if isinstance(v, (int, float))] + + def avg(items, fn): + vals = values(items, fn) + return statistics.mean(vals) if vals else None + + def spread(items, fn, digits=1): + vals = values(items, fn) + if not vals: + return "—" + return "%s (%s…%s)" % (fmt(statistics.mean(vals), digits), fmt(min(vals), digits), fmt(max(vals), digits)) + + lat = lambda q: (lambda x: x["load"]["latency_ms_plain_ok"][q]) # noqa: E731 + cpu = lambda *svc: (lambda x: sum(x["cpu_pct_avg"].get(s, 0) for s in svc)) # noqa: E731 + delta_of = lambda key: (lambda x: x["telemetry_delta"].get(key)) # noqa: E731 + traces = lambda key: (lambda x: x["traces_in_tempo"].get(key)) # noqa: E731 + faults = lambda key: (lambda x: x["load"]["faults_sent"][key]) # noqa: E731 + names = [n for n in CONFIGS if n in by] + + lines = ["Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.", "", + "| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |", + "|---|---|---|---|---|---|---|---|---|"] + for n in names: + i = by[n] + lines.append("| %s | %d | %s | %s | %s | %s | %s | %s | %s |" % ( + n, len(i), fmt(avg(i, lambda x: x["load"]["successful_rps"]), 1), + fmt(avg(i, lambda x: x["load"]["server_errors_unexpected"])), spread(i, lat("p50")), spread(i, lat("p95")), + spread(i, lat("p99")), spread(i, cpu("api-1", "api-2")), spread(i, cpu("worker")))) + lines += ["", "| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |", + "|---|---|---|---|---|---|---|---|"] + for n in names: + i = by[n] + mem = lambda s: (lambda x: x["mem_mib_max"].get(s)) # noqa: E731 + lines.append("| %s | %s | %s | %s | %s | %s | %s | %s |" % ( + n, spread(i, cpu("otel-collector"), 2), spread(i, cpu("tempo"), 2), spread(i, cpu("loki"), 2), + spread(i, cpu("prometheus"), 2), fmt(avg(i, mem("api-1"))), fmt(avg(i, mem("otel-collector"))), + fmt(avg(i, mem("tempo"))))) + lines += ["", "| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |", + "|---|---|---|---|---|---|---|"] + for n in names: + i = by[n] + kib = lambda key: (avg(i, delta_of(key)) / 1024 if avg(i, delta_of(key)) is not None else None) # noqa: E731 + lost = avg(i, lambda x: (x["telemetry_delta"].get("col_spans_refused") or 0) + (x["telemetry_delta"].get("col_spans_dropped") or 0)) + lines.append("| %s | %s | %s | %s | %s | %s | %s |" % ( + n, fmt(avg(i, delta_of("col_spans_in"))), fmt(avg(i, delta_of("tempo_spans_in"))), fmt(lost), + fmt(kib("col_otlp_bytes_in")), fmt(kib("tempo_bytes_in")), fmt(kib("loki_bytes_in")))) + lines += ["", "Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.", "", + "| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |", + "|---|---|---|---|---|---|"] + for n in names: + i = by[n] + total = lambda fn: sum(values(i, fn)) # noqa: E731 + m500 = [metric_500s(x) for x in i] + lines.append("| %s | %d / %d | %d / %d | %d / %d | %d / %d | %s / %d |" % ( + n, total(traces("server_api")), total(lambda x: x["load"]["requests"]), + total(traces("slow")), total(faults("slow")), total(traces("error")), total(faults("error")), + total(traces("pipeline_consumer")), total(lambda x: x["load"]["by_kind"]["upload"]["n"]), + "—" if None in m500 else "%.0f" % sum(m500), total(faults("error")))) + text = "\n".join(lines) + "\n" + (OUT / "summary.md").write_text(text, encoding="utf-8") + print(text) + + +HARDWARE = {} + + +def main(): + p = argparse.ArgumentParser() + sub = p.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("run") + r.add_argument("--configs", default="off,on100,head10,tail") + r.add_argument("--repeats", type=int, default=2) + r.add_argument("--rate", type=float, default=60) + r.add_argument("--duration", type=float, default=90) + sub.add_parser("summary") + args = p.parse_args() + if args.cmd == "run": + info = subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} {{.MemTotal}}"], + capture_output=True, text=True).stdout.split() + HARDWARE.update({"docker": info[0], "vm_cpus": int(info[1]), "vm_mem_gib": round(int(info[2]) / 2**30, 1)}) + cmd_run(args) + else: + cmd_summary(args) + + +if __name__ == "__main__": + main() diff --git a/student-tests/variants_check.py b/student-tests/variants_check.py new file mode 100644 index 0000000..6ab763c --- /dev/null +++ b/student-tests/variants_check.py @@ -0,0 +1,126 @@ +"""Variant rules and worker resource limits. Runs INSIDE the worker container (Pillow + app code): + +docker compose exec -T --index 1 worker python - < student-tests/variants_check.py +""" +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +import time + +from PIL import Image + +sys.path.insert(0, "/app") +from app import config # noqa: E402 + +FAILS = [] + + +def check(cond, msg): + print(("ok " if cond else "FAIL ") + msg, flush=True) + if not cond: + FAILS.append(msg) + + +def run_child(path, mem_mb=config.WORKER_CHILD_MEMORY_MB, cpu=config.JOB_TIMEOUT_SECONDS, max_pixels=config.MAX_IMAGE_PIXELS): + out = tempfile.mkdtemp() + + def limits(): + resource.setrlimit(resource.RLIMIT_AS, (mem_mb << 20, mem_mb << 20)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + + start = time.monotonic() + done = subprocess.run([sys.executable, "-m", "app.imageproc", path, out, str(max_pixels)], capture_output=True, + preexec_fn=limits, timeout=120, env={"PATH": os.environ["PATH"], "MALLOC_ARENA_MAX": "2"}) + usage = resource.getrusage(resource.RUSAGE_CHILDREN) + produced = {v["name"]: v for v in json.loads(done.stdout)} if done.returncode == 0 else {} + return done.returncode, produced, time.monotonic() - start, usage.ru_maxrss / 1024 + + +def save(img, fmt, **kw): + path = tempfile.mktemp(suffix="." + fmt.lower()) + img.save(path, fmt, **kw) + return path + + +def dims(v): + return v["width"], v["height"] + + +def real(v): + with Image.open(v["file"]) as im: + return im.format, im.size, "exif" in im.info, im + + +# 1. fixture 320x200 -> 256x160, crop 128, optimized not upscaled +rc, p, _, _ = run_child("/app/tests/fixtures/sample.png" if os.path.exists("/app/tests/fixtures/sample.png") else + save(Image.new("RGB", (320, 200), "red"), "PNG")) +check(rc == 0 and dims(p["thumbnail"]) == (256, 160), "320x200 -> thumbnail 256x160") +check(dims(p["crop"]) == (128, 128), "crop 128x128") +check(dims(p["optimized"]) == (320, 200) and real(p["optimized"])[0] == "WEBP", "optimized WebP 320x200, no upscale") +check(real(p["thumbnail"])[0] == "JPEG" and real(p["crop"])[0] == "JPEG", "thumbnail/crop are JPEG") + +# 2. small source: no upscale for thumbnail, upscale allowed for crop +rc, p, _, _ = run_child(save(Image.new("RGB", (40, 10), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (40, 10) and dims(p["crop"]) == (128, 128), "40x10: thumbnail 40x10, crop 128x128") + +# 3. extreme aspect ratio: floor rounding, minimum 1 px +rc, p, _, _ = run_child(save(Image.new("RGB", (3000, 2), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (256, 1) and dims(p["optimized"]) == (1280, 1), "3000x2 -> 256x1 / 1280x1 (min 1px)") +rc, p, _, _ = run_child(save(Image.new("RGB", (1001, 999), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (256, 255), "1001x999 -> 256x255 (floor)") + +# 4. EXIF orientation 6 (rotate 90): stored 400x200 -> upright 200x400 -> thumbnail 128x256; EXIF stripped +exif = Image.Exif() +exif[0x0112] = 6 +img = Image.new("RGB", (400, 200), "white") +img.paste((255, 0, 0), (0, 0, 200, 200)) # left half red +rc, p, _, _ = run_child(save(img, "JPEG", exif=exif.tobytes(), quality=95)) +fmt, size, has_exif, im = real(p["thumbnail"]) +check(size == (128, 256), "EXIF orientation applied: 400x200 (o=6) -> 128x256") +with Image.open(p["thumbnail"]["file"]) as t: + top = t.convert("RGB").getpixel((64, 20)) +check(top[0] > 200 and top[1] < 80, "rotated content: red half on top after orientation 6") +check(not has_exif and "exif" not in Image.open(p["optimized"]["file"]).info, "EXIF removed from thumbnail/optimized") + +# 5. alpha composited on white for JPEG, kept for WebP +rgba = Image.new("RGBA", (100, 100), (0, 0, 0, 0)) +rc, p, _, _ = run_child(save(rgba, "PNG")) +with Image.open(p["thumbnail"]["file"]) as t: + px = t.convert("RGB").getpixel((50, 50)) +check(min(px) >= 250, "transparent PNG -> white JPEG thumbnail") +check(Image.open(p["optimized"]["file"]).mode == "RGBA", "WebP optimized keeps alpha") + +# 6. 16-bit grayscale PNG, CMYK JPEG, palette PNG +rc16, _, _, _ = run_child(save(Image.new("I;16", (300, 300), 40000), "PNG")) +rcc, _, _, _ = run_child(save(Image.new("CMYK", (300, 300), (0, 255, 0, 0)), "JPEG")) +rcp, _, _, _ = run_child(save(Image.new("P", (300, 300), 3), "PNG", transparency=3)) +check(rc16 == 0 and rcc == 0 and rcp == 0, "16-bit PNG, CMYK JPEG, palette+transparency PNG are processed") + +# 7. limits: pixels (exit 4), not an image (exit 3), memory (exit 5), CPU (SIGXCPU) +rc, _, _, _ = run_child(save(Image.new("RGB", (100, 100)), "PNG"), max_pixels=5000) +check(rc == 4, "pixel limit enforced before decode -> image_too_large (exit 4)") +junk = tempfile.mktemp() +open(junk, "wb").write(b"\x89PNG\r\n\x1a\n" + os.urandom(2000)) +rc, _, _, _ = run_child(junk) +check(rc == 3, "corrupted PNG -> invalid_image (exit 3)") +big = save(Image.new("RGB", (6000, 6000), "green"), "PNG") +rc, _, _, _ = run_child(big, mem_mb=160) +check(rc == 5, "36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got %s" % rc) + +# 8. largest allowed inputs under the real limits: time and child peak RSS +for name, w, h, fmt in (("png", 7900, 5000, "PNG"), ("jpeg", 7900, 5000, "JPEG")): + noise = Image.effect_noise((w // 10, h // 10), 60).convert("RGB").resize((w, h)) + path = save(noise, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 1})) + rc, p, secs, rss = run_child(path) + check(rc == 0 and dims(p["thumbnail"]) == (256, 162), "%.1f Mpx %s (%d MB file) ok in %.1fs, peak child RSS %.0f MiB" + % (w * h / 1e6, name.upper(), os.path.getsize(path) >> 20, secs, rss)) + if fmt == "PNG": + rc, _, _, _ = run_child(path, cpu=0) + check(rc == -24, "CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got %s" % rc) + +print("RESULT:", "PASS" if not FAILS else "FAIL %s" % FAILS) +sys.exit(1 if FAILS else 0) -- 2.54.0