forked from mtusi2026/mtusi2026lab7
лр7: Docker-прогон, evidence, исправления
This commit is contained in:
+61
-25
@@ -1,41 +1,77 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# lab7: здесь НЕТ секретов и путей к файлам с паролями. Учётные данные процессов (PostgreSQL, Redis, S3)
|
||||
# выдаёт OpenBao по сертификату сервиса; адреса, роли и TLS-пути задаёт compose.yaml для каждого сервиса.
|
||||
# Соответствие переменных skeleton -> решение — REPORT.md, «Соответствие .env.example».
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_PORT=8443
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
SESSION_STORE=redis
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=https://s3:9000
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
|
||||
# S3 (SeaweedFS 4.52): HTTPS, у api/worker/janitor свои ключи с правами по prefix (из OpenBao)
|
||||
S3_ENDPOINT=https://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
|
||||
# lab5: broker/worker. Брокер — amqps://rabbitmq:5671/messenger, вход по сертификату (EXTERNAL); задаётся в compose.
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=15
|
||||
JOB_BACKOFF_BASE_SECONDS=2
|
||||
JOB_BACKOFF_MAX_SECONDS=60
|
||||
WORKER_MEMORY_LIMIT_MB=512
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
TLS_CA_FILE=/run/secrets/ca.crt
|
||||
TLS_CERT_FILE=/run/secrets/service.crt
|
||||
TLS_KEY_FILE=/run/secrets/service.key
|
||||
BAO_ADDR=https://openbao:8200
|
||||
BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
PIPELINE_VERSION=1
|
||||
|
||||
# Pools and timeouts (как в lab3–6)
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_WAIT_SECONDS=3
|
||||
DB_CONNECT_TIMEOUT_SECONDS=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
REDIS_POOL_MAX=20
|
||||
REDIS_TIMEOUT_MS=500
|
||||
SHUTDOWN_DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=10
|
||||
S3_CONNECT_TIMEOUT_SECONDS=1
|
||||
S3_READ_TIMEOUT_SECONDS=5
|
||||
S3_MAX_ATTEMPTS=2
|
||||
S3_POOL_MAX=20
|
||||
UPLOAD_TIMEOUT_SECONDS=60
|
||||
UPLOAD_IDLE_TIMEOUT_SECONDS=10
|
||||
DECODE_CONCURRENCY=2
|
||||
ATTACHMENT_GRACE_SECONDS=3600
|
||||
JANITOR_INTERVAL_SECONDS=300
|
||||
|
||||
# lab6: OpenTelemetry (service.name и mTLS-сертификат для каждого сервиса задаёт compose.yaml)
|
||||
OTEL_SDK_DISABLED=false
|
||||
|
||||
# lab7: OpenBao. Адрес/CA/роль задаёт compose.yaml (BAO_ADDR=https://openbao:8200, BAO_ROLE=<CN сервиса>).
|
||||
# Как часто процесс перечитывает свои секреты (подхват ротации) и продлевает токен.
|
||||
SECRETS_REFRESH_SECONDS=30
|
||||
SECRETS_STARTUP_TIMEOUT_SECONDS=60
|
||||
# mTLS: CRL учебного CA перечитывается API раз в N секунд (отзыв identity без рестарта).
|
||||
CRL_RELOAD_SECONDS=5
|
||||
|
||||
# lab7: Argon2id (не ниже m=19456 KiB, t=2, p=1). Выбор 32 MiB / t=2 / p=1 — по замеру, REPORT.md.
|
||||
ARGON2_MEMORY_KIB=32768
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
# Бюджет хеширования реплики: не больше N хешей одновременно, ожидание слота не дольше — иначе 503 + Retry-After.
|
||||
ARGON2_MAX_CONCURRENCY=2
|
||||
ARGON2_QUEUE_SECONDS=2
|
||||
|
||||
# lab7: общий (Redis) лимит попыток входа/регистрации, фиксированное окно.
|
||||
LOGIN_RATE_WINDOW_SECONDS=60
|
||||
LOGIN_RATE_PER_USERNAME=10
|
||||
LOGIN_RATE_PER_CLIENT=600
|
||||
REGISTER_RATE_PER_CLIENT=600
|
||||
# Ограничение размера JSON-тела (вложения — MAX_UPLOAD_BYTES).
|
||||
MAX_JSON_BYTES=65536
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
*.sh text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.example text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.json text eol=lf
|
||||
*.py text eol=lf
|
||||
*.md text eol=lf
|
||||
# Конфиги, которые читают Linux-контейнеры (OpenBao, SeaweedFS, RabbitMQ), и файлы сборки
|
||||
*.hcl text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.txt text eol=lf
|
||||
*.http text eol=lf
|
||||
infra/rabbitmq-enabled-plugins text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
Makefile text eol=lf
|
||||
.gitignore text eol=lf
|
||||
.dockerignore text eol=lf
|
||||
*.jpg binary
|
||||
*.png binary
|
||||
@@ -1,18 +1,18 @@
|
||||
## Лабораторная 7
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Бабиченко Максим, БВТ2303
|
||||
Целевая оценка (3/4/5): 4
|
||||
Трек 5, если заявлен: не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: перенесено всё решение lab6 (API lab1–5, outbox/dispatcher/worker/janitor, OpenTelemetry + Collector/Prometheus/Loki/Tempo/Grafana, тесты lab1–6). Добавлены: учебный PKI оператора (`scripts/pki.py`: CA, leaf на каждую identity, CRL, ротация/отзыв); HTTPS-ingress и mTLS LB→API с allowlist CN=lb и CRL (в т.ч. закрытие открытых соединений отозванных сертификатов); TLS на всех связях — PostgreSQL (verify-full, hostssl+SCRAM, 4 роли с GRANT), Redis (TLS+client cert+ACL), S3 (HTTPS, ключи по prefix, mTLS внутреннего gRPC), RabbitMQ (AMQPS, EXTERNAL по CN), Collector (mTLS-шлюз с allowlist CN), Prometheus/Loki/Tempo/Grafana (mTLS); OpenBao 2.6.2 не dev-mode (file storage, init/unseal оператором, cert auth по CN, политики по пути, токены 15 мин/1 ч, renew/relogin, работа при sealed); Argon2id 32 MiB/t=2/p=1 с rehash, бюджет хеширования на реплику, общий rate limit в Redis (429 + Retry-After), лимиты тела; hardening всех контейнеров; скан зависимостей (Pillow → 13.0.0) и образов.
|
||||
Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`pki.py init`, `init_secrets.py`, `docker compose up -d openbao`, `bao.py init/unseal/configure`, `docker compose up -d --build --wait`, `collect_evidence.py`, `scan.py`).
|
||||
Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев»; `evidence/native-sandbox/` (прогон без Docker), `evidence/` (Docker-прогон), `contracts/trust-matrix.md`, `security/THREAT_MODEL.md`, `security/scan-triage.md`.
|
||||
Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги» (один хост, доли unseal в файле оператора, CRL не проверяют RabbitMQ/Redis/PostgreSQL, Loki/Tempo не разделяют читателя и писателя).
|
||||
Источники и использование ИИ: REPORT.md, шапка (документация OWASP, OpenBao, RabbitMQ, PostgreSQL, Redis, SeaweedFS, Prometheus, Grafana; ИИ-ассистент Claude).
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод (песочница: `evidence/native-sandbox/`; Docker-прогон: `evidence/`).
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены (Docker-прогон: `evidence/*.json`).
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
# Образ приложения lab7 (api, worker, dispatcher, janitor, migrate, s3-init, ops.py — разные команды).
|
||||
# В образе нет ни одного секрета/ключа: сертификаты и CA монтируются при запуске (secrets/pki/*),
|
||||
# учётные данные процесс получает из OpenBao по своему сертификату.
|
||||
FROM python:3.11.13-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Security-обновления Debian базового слоя: скан Docker-прогона (security/scan-triage.md) нашёл в python:3.11.13-slim
|
||||
# OpenSSL 3.5.1 (CRITICAL/HIGH), zlib, sqlite, libuuid — их загружает сам CPython (ssl, zlib, uuid) на каждом TLS-соединении.
|
||||
RUN apt-get update && apt-get upgrade -y --no-install-recommends && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
# После установки зависимостей pip/setuptools/wheel не нужны в рантайме — удаляем (меньше поверхность и находок скана).
|
||||
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt \
|
||||
&& (pip uninstall -y pip setuptools wheel || true)
|
||||
|
||||
COPY main.py settings.py errors.py db.py dbconn.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py \
|
||||
s3_init.py pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py telemetry.py \
|
||||
secretstore.py tlsutil.py ./
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
# Фиксированный uid/gid (10001): не root; файлы кода принадлежат root и только читаются.
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin app \
|
||||
&& mkdir -p /faults && chown app:app /faults
|
||||
# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher).
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
EXPOSE 8443
|
||||
|
||||
# Проверка здоровья — loopback listener 127.0.0.1:8081 (только /health/*), бизнес-API там нет.
|
||||
HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=4)"
|
||||
|
||||
CMD ["python", "main.py"]
|
||||
@@ -1,37 +1,325 @@
|
||||
# Отчёт — лабораторная 7
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Бабиченко Максим, БВТ2303
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не выбран
|
||||
- Commit/PR: <<ссылка на MR после публикации>>
|
||||
- Использованные источники, библиотеки и помощь ИИ: всё из lab6 (FastAPI, psycopg 3, pika, boto3, Pillow, OpenTelemetry SDK, RabbitMQ 4.1, SeaweedFS, PostgreSQL 17, Redis 7.4, HAProxy 3.2, OTel Collector, Prometheus, Loki, Tempo, Grafana). Новое: `cryptography` 50.0.2 (выпуск сертификатов оператором, разбор и проверка подписи CRL в API), образ `openbao/openbao:2.6.2`, SeaweedFS обновлён до 4.52 (TLS шлюза S3, права по prefix, mTLS внутреннего gRPC), Pillow 12.3.0 → 13.0.0 и security-обновления Debian в образе приложения (скан). Документация: OWASP Password Storage и File Upload Cheat Sheets, RFC 9106 (Argon2), OpenBao (cert auth, policies, KV v2, seal/unseal, production hardening), RabbitMQ TLS и `rabbitmq_auth_mechanism_ssl`, PostgreSQL `pg_hba`/SSL/SCRAM, Redis TLS и ACL, SeaweedFS `security.toml`, Prometheus web configuration (`client_allowed_sans`), Grafana/Tempo/Loki TLS, Docker security, pip-audit, Trivy. При написании кода, конфигураций и тестов использовался ИИ-ассистент (Claude); весь код я прогнал и могу объяснить и изменить на защите.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Проверяю на Windows (PowerShell, Docker Desktop). На хосте нужен Python 3.11+: тесты — только стандартная библиотека, операторские скрипты PKI — пакет `cryptography` (`requirements-ops.txt`). Перед запуском остановить стенд lab6 (`docker compose down` в его папке). Имя проекта Compose — `messenger-lab7`, volumes свои.
|
||||
|
||||
```powershell
|
||||
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab7.git
|
||||
cd mtusi2026lab7
|
||||
git checkout solution/bvt2303-babichenko
|
||||
$env:PYTHONUTF8 = "1"
|
||||
python -m pip install -r requirements-ops.txt # cryptography — только для scripts/pki.py
|
||||
Copy-Item .env.example .env
|
||||
|
||||
# 1. Оператор: учебный CA и сертификаты сервисов (ключ CA остаётся в secrets/pki/ca, никуда не монтируется)
|
||||
python scripts/pki.py init
|
||||
# 2. Оператор: пароли/ключи стенда (серверные конфиги + файл для OpenBao), значения не печатаются
|
||||
python scripts/init_secrets.py
|
||||
# 3. OpenBao: старт, инициализация (3 доли, порог 2), распечатка, настройка (KV, cert auth, политики, секреты)
|
||||
docker compose up -d openbao
|
||||
python scripts/bao.py init
|
||||
python scripts/bao.py unseal
|
||||
python scripts/bao.py configure
|
||||
# 4. Весь стенд
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose up -d --build --wait
|
||||
docker compose ps -a # migrate, s3-init, telemetry-init: Exited (0); остальные running/healthy
|
||||
|
||||
# 5. Проверки
|
||||
python tests/smoke.py --base-url https://localhost:8443 --ca-file secrets/pki/public/ca.crt # make test CA_FILE=...
|
||||
python tests/security.py --base-url https://localhost:8443 --ca-file secrets/pki/public/ca.crt # make test-security
|
||||
python student-tests/collect_evidence.py # все шаги lab7 + регрессия lab1–6 -> evidence/, таблицы в REPORT.md
|
||||
python scripts/scan.py # pip-audit + Trivy по образам -> evidence/scan/
|
||||
```
|
||||
|
||||
| Что | Где |
|
||||
| --- | --- |
|
||||
| API (единственная точка входа) | https://localhost:8443, доверять `secrets/pki/public/ca.crt` (сертификат ingress: SAN localhost, 127.0.0.1) |
|
||||
| Grafana | https://localhost:3000, логин `admin`, пароль — `secrets/grafana_admin_password`. Браузеру нужен наш CA: `Import-Certificate -FilePath secrets\pki\public\ca.crt -CertStoreLocation Cert:\CurrentUser\Root` (по желанию; проверку не отключаем) |
|
||||
| Prometheus / Loki / Tempo | https://localhost:9090, :3100, :3200 — только с клиентским сертификатом оператора: `curl.exe --cacert secrets/pki/public/ca.crt --cert secrets/pki/operator/tls.crt --key secrets/pki/operator/tls.key https://localhost:9090/-/ready` |
|
||||
| Статистика HAProxy | https://localhost:8404/stats — тот же сертификат оператора |
|
||||
| OpenBao (оператор) | https://127.0.0.1:8200, `python scripts/bao.py status` |
|
||||
|
||||
Отдельные проверки lab7 (make на Windows нет):
|
||||
|
||||
| Команда | Что проверяет |
|
||||
| --- | --- |
|
||||
| `python student-tests/test_tls_matrix.py` | вся trust matrix изнутри сети (контейнер `probe`): разрешённая identity проходит, без сертификата / чужой CA / чужая identity нашего CA / истёкший / отозванный — отказ; plaintext не отвечает |
|
||||
| `python student-tests/test_least_privilege.py` | api/worker/dispatcher/janitor своими учётными данными: GRANT PostgreSQL, prefix S3, политики OpenBao, ACL Redis |
|
||||
| `python student-tests/test_lab7_security.py` | только HTTPS, заголовки, посторонний пользователь, logout, общий rate limit, лимиты тела, Argon2id, rehash, отсутствие утечек |
|
||||
| `python student-tests/lifecycle.py rotate-secret rotate-leaf token-expiry sealed` | ротация секрета и leaf, отзыв, истечение токенов, sealed OpenBao — под нагрузкой |
|
||||
| `python student-tests/leak_check.py` | пароль/токен/Authorization не попадают в логи контейнеров, Loki, Tempo, метки Prometheus |
|
||||
| `python student-tests/argon2_bench.py`, `python student-tests/login_dos.py` | замер Argon2id в контейнере api; бюджет защиты от login DoS |
|
||||
| `python student-tests/check_hardening.py --runtime` | privileged/host/docker.sock, capabilities, no-new-privileges, read_only, не root, лимиты, порты |
|
||||
| `python student-tests/run_persistence.py`, `python student-tests/run_failover.py` | публичные `test-persistence` (hook `scripts\restart.cmd`, включая рестарт и unseal OpenBao) и `test-failover` (hook `scripts\stop-one.cmd`) по HTTPS |
|
||||
|
||||
**После перезагрузки хоста / рестарта OpenBao** он запечатан: `python scripts/bao.py unseal`. До этого уже работающих процессов нет, новые не стартуют (ждут до 60 с и перезапускаются Docker) — после unseal стенд поднимается сам. **Остановка без потери данных:** `docker compose down` (volumes `openbaodata`, `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults`, `promdata`, `lokidata`, `tempodata`, `grafanadata` остаются). **Намеренное удаление учебных данных:** `docker compose down -v`, удалить папку `secrets` (CA, ключи, доли unseal, пароли) и пройти шаги 1–4 заново.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент --HTTPS 127.0.0.1:8443--> lb (HAProxy) ==mTLS, клиент CN=lb==> api x2 (FastAPI)
|
||||
api --TLS verify-full + SCRAM, роль messenger_api--> postgres api --TLS+mTLS + ACL user api--> redis
|
||||
api --HTTPS + ключ api (staging/ write, attachments/ read)--> s3 api ==cert auth CN=api==> openbao
|
||||
api/worker/dispatcher/janitor ==mTLS, CN из allowlist==> otel-gate --127.0.0.1--> otel-collector
|
||||
dispatcher ==AMQPS EXTERNAL CN=dispatcher==> rabbitmq <==AMQPS EXTERNAL CN=worker== worker
|
||||
otel-collector ==mTLS CN=otel-collector==> prometheus | loki | tempo <==mTLS CN=grafana== grafana
|
||||
```
|
||||
|
||||
**Что перенесено из lab6:** весь API (lab1–5), outbox/dispatcher/worker/janitor, телеметрия (SDK, Collector, Prometheus/Loki/Tempo/Grafana, dashboard, алерты, runbooks), тесты lab1–6. **Что изменено в lab7:**
|
||||
|
||||
| Область | Решение | Где |
|
||||
| --- | --- | --- |
|
||||
| PKI | Учебный CA оператора (ECDSA P-256), leaf на каждую identity (30 суток, CN + URI SAN `spiffe://messenger.lab/<cn>`), CRL, ротация/отзыв, тестовые сертификаты (чужой CA, intruder, истёкший) | `scripts/pki.py` |
|
||||
| Внешний TLS | HAProxy: только HTTPS :8443, TLS 1.2+, без tickets, HSTS, nosniff, без `Server`; `X-Forwarded-For` перезаписывается; тело > 11 MiB → 413 | `infra/haproxy.cfg` |
|
||||
| LB → API | mTLS: HAProxy проверяет сертификат API (CA, CRL, имя api) и предъявляет `lb`; API (`tlsutil.py`, свой протокол uvicorn) — цепочка/срок (OpenSSL) + **allowlist CN** и CRL до чтения запроса; CRL перечитывается раз в 5 с, открытые соединения отозванных сертификатов закрываются. Health — отдельный listener 127.0.0.1:8081 | `tlsutil.py`, `main.py` |
|
||||
| OpenBao | Не dev-mode: file storage в volume, TLS-listener, init/unseal оператором, KV v2 `messenger/`, **cert auth** (роль = CN, только наш CA, CRL), 6 политик по одному пути, токены 15 мин / max 1 ч. Клиент приложения: вход, renew, relogin, перечитывание KV раз в 30 с, работа на кэше при sealed | `infra/openbao/`, `scripts/bao.py`, `secretstore.py` |
|
||||
| PostgreSQL | `ssl=on`, `pg_hba`: только `hostssl` + SCRAM; 4 роли приложения с GRANT по запросам (`006_roles.sql`), пароли из OpenBao на каждое соединение, migrate ставит их SCRAM-verifier'ом | `infra/postgres/pg_hba.conf`, `dbconn.py`, `migrate.py` |
|
||||
| Redis | Только TLS, клиентский сертификат обязателен, ACL: `default off`, `api` — `session:*`/`ratelimit:*` и 13 команд; пароль через credential provider (ротация без рестарта) | `infra/redis.conf`, `auth.py` |
|
||||
| S3 | SeaweedFS 4.52: HTTPS :8333, внутренние порты на loopback, gRPC :18333 — mTLS только CN=s3, IAM/Iceberg/Lance выключены; ключи api/worker/janitor/s3-init с правами по prefix | `infra/seaweedfs/security.toml`, `scripts/init_secrets.py` |
|
||||
| RabbitMQ | Только AMQPS :5671, `verify_peer`, EXTERNAL (пользователь = CN), пользователи без паролей, distribution/epmd на loopback; плагины — только `rabbitmq_auth_mechanism_ssl` | `infra/rabbitmq.conf`, `broker.py` |
|
||||
| Телеметрия | otel-gate (HAProxy) — mTLS и allowlist CN приложений перед Collector (Collector на 127.0.0.1); Collector → Tempo/Loki/Prometheus по mTLS; Prometheus `client_allowed_sans`; Loki gRPC на loopback, Tempo внутренний gRPC по TLS; Grafana по HTTPS, источники — mTLS | `infra/otel/`, `infra/prometheus/web.yml`, `infra/loki`, `infra/tempo`, `infra/grafana` |
|
||||
| Пароли и вход | Argon2id m=32 MiB, t=2, p=1 (замер ниже), rehash при входе, ≤2 хеша на реплику одновременно (иначе 503 + Retry-After), общий rate limit в Redis (429 + Retry-After), JSON ≤ 64 KiB (в т.ч. chunked) | `auth.py`, `main.py` |
|
||||
| Hardening | Все сервисы: cap_drop ALL, no-new-privileges, read_only + tmpfs, не root, лимиты CPU/RAM/PID, нет privileged/host/docker.sock; объяснённые исключения (RabbitMQ записываемый корень, telemetry-init CHOWN); SeaweedFS — uid 1000 образа (в Docker-прогоне исключение uid 0 снято). Наружу — только 127.0.0.1 | `compose.yaml`, `Dockerfile`, `student-tests/check_hardening.py` |
|
||||
| X-Instance-Id | Случайный alias процесса (12 hex), а не ID контейнера; `stop_one.py` находит реплику по файлу alias | `settings.py`, `scripts/stop_one.py` |
|
||||
|
||||
**Источник истины** не менялся: PostgreSQL (данные), S3 (байты), Redis (сессии, счётчики лимитов), OpenBao (учётные данные сервисов). **Транзакционные границы** — как в lab5/6 (outbox). **Готовность (`/health/ready`)**: PostgreSQL + Redis + секреты загружены; OpenBao в readiness не входит — его недоступность не останавливает уже работающую реплику (см. «Эксперименты»).
|
||||
|
||||
### Соответствие `.env.example`
|
||||
|
||||
| Переменная skeleton | В решении |
|
||||
| --- | --- |
|
||||
| `APP_PORT=8080` | API слушает **8443** (только mTLS); наружу — HAProxy 127.0.0.1:8443 |
|
||||
| `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE`, `S3_*_FILE` | не используются: адреса — `DB_HOST/DB_USER/REDIS_HOST/REDIS_USER/BROKER_URL/S3_ENDPOINT` в compose, учётные данные — из OpenBao (`messenger/<сервис>`); брокер — вход по сертификату |
|
||||
| `S3_ENDPOINT=https://s3:9000` | `https://s3:8333` (SeaweedFS S3) |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318` | так и есть (alias otel-gate) + `OTEL_EXPORTER_OTLP_CERTIFICATE/CLIENT_CERTIFICATE/CLIENT_KEY` |
|
||||
| `PUBLIC_BASE_URL` | `https://localhost:8443` (тесты, ingress) |
|
||||
| `TLS_CA_FILE`, `TLS_CERT_FILE`, `TLS_KEY_FILE` | `/run/tls-ca/ca.crt`, `/run/tls/tls.crt`, `/run/tls/tls.key` (+ `TLS_CRL_FILE=/run/tls-ca/crl.pem`) |
|
||||
| `BAO_ADDR`, `BAO_CACERT` | `BAO_ADDR=https://openbao:8200`; CA — тот же `TLS_CA_FILE`; роль — `BAO_ROLE` = CN сертификата |
|
||||
| `ARGON2_MEMORY_KIB=19456`, `ARGON2_ITERATIONS=2`, `ARGON2_PARALLELISM=1` | `32768 / 2 / 1` (+ `ARGON2_MAX_CONCURRENCY=2`, `ARGON2_QUEUE_SECONDS=2`) |
|
||||
| — | новые: `SECRETS_REFRESH_SECONDS`, `CRL_RELOAD_SECONDS`, `LOGIN_RATE_*`, `REGISTER_RATE_PER_CLIENT`, `MAX_JSON_BYTES` |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
Где проверено: **Docker-стенд** на машине автора (Windows 10, Docker Desktop, 16 CPU / 15 GiB VM): стенд поднят с нуля шагами раздела «Запуск», затем `python student-tests/collect_evidence.py` — **все 46 шагов PASS** (`evidence/summary.md`), результаты в `evidence/*.txt|json`, замеры — блок ниже. Ранее решение проверялось в **нативной песочнице** без Docker (`evidence/native-sandbox/`, оставлена для сравнения): тот же стенд процессами, конфиги из `infra/`, вместо `docker` — подмена, управляющая процессами. Ошибки, которые песочница не поймала и которые исправлены по Docker-прогону, — в «Ограничениях». Отличия песочницы: PostgreSQL 16 вместо 17, Redis 7.0 вместо 7.4, RabbitMQ 4.4 вместо 4.1.4, HAProxy 2.8 вместо 3.2, **HashiCorp Vault 2.1.2 как API-совместимая замена OpenBao 2.6.2** (тот же HTTP API: sys/init, unseal, cert auth, KV v2, policies), Prometheus 2.45 (только web-config TLS), Collector — заглушка OTLP за настоящим otel-gate; Loki/Tempo/Grafana не запускались (их TLS-конфиги проверяются только в Docker).
|
||||
|
||||
### Уровень 3
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker) | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke по HTTPS с проверяемым CA и hostname | `make test CA_FILE=secrets/pki/public/ca.crt` | PASS: lab7 public API scenario (регистрация, вход, чат, сообщения, пагинация, идемпотентность, вложение → thumbnail, logout) | `evidence/smoke.txt` |
|
||||
| Внешний TLS: недоверенный CA отвергается | `make test-security`; `test_lab7_security.py test_https_only_and_certificate_checks` | PASS: пустой trust store, чужой CA → `SSLCertVerificationError`; имя `evil.example` → Hostname mismatch; plaintext на 8443 — без ответа приложения; 8080 — не API | `security.txt`, `lab7-security.txt` |
|
||||
| Предыдущие возможности через HTTPS | регрессия lab1–5: `test_api`, `test_race`, `test_idempotency`, `test_balancing`, `test_attachments`, `test_consistency`, `test_jobs`, `test_janitor_replay`, `test_failures`, `test_db`, `redis_info`, `explain`, `load` | все PASS (см. сводку) | `evidence/*.txt` |
|
||||
| Argon2id: соль, параметры, ≥19 MiB/t=2/p=1, обоснование замером | `test_password_storage_argon2id`, `argon2_bench.py` | `$argon2id$v=19$m=32768,t=2,p=1$<16-байтная соль>$…`; одинаковый пароль → разные хеши; замер — «Эксперименты» | `lab7-security.txt`, `argon2-bench.json` |
|
||||
| LB→API mTLS, отказ без cert / чужой CA / неверная identity; нет обхода | `test_tls_matrix.py lb-api` | lb → 200; без сертификата, чужой CA с CN=lb, истёкший lb, intruder (наш CA), worker, api → соединение закрыто; клиент с чужим CA не принимает сервер; plaintext — нет ответа; порт API не опубликован | `tls-matrix.json` |
|
||||
| Остальные связи — TLS с проверкой сервера, раздельные учётные данные и минимальные права | `test_tls_matrix.py`, `test_least_privilege.py`, `contracts/trust-matrix.md` | 78 проверок matrix (12 связей, включая Loki/Tempo/Grafana и внутренний gRPC Tempo), 83 проверки прав — все PASS | `tls-matrix.json`, `least-privilege.json` |
|
||||
| OpenBao не dev-mode, persistent storage, сервисная аутентификация, scoped policy, без root token в приложении | `bao.py init/unseal/configure`; `test_least_privilege.py` (openbao); `run_persistence.py` (рестарт OpenBao + unseal, данные на месте) | cert auth по CN; чужие пути, запись KV/политики, список auth, создание токена → 403; worker как роль api → 400 | `least-privilege.txt`, `tls-matrix.txt`, `persistence.txt` |
|
||||
| Контейнеры: без privileged/host/docker.sock, app/worker non-root, без лишних capabilities, лимиты; наружу только TLS-ingress и локальные админ-порты; секреты не в Git/образах/телеметрии | `check_hardening.py --runtime`, `check_stack.py`, `leak_check.py`, `.gitignore`/`.dockerignore` | статически 183 PASS, 4 объяснённых исключения, 0 нарушений + `docker inspect` всех контейнеров; маркерные пароль/токен — 0 в логах и OTLP | `hardening.json`, `leak-check.json` |
|
||||
| 429 + Retry-After, лимит размера, посторонний пользователь, logout/expiry, недоверенный TLS | `test_lab7_security.py`, `test_expiry.py` (lab2) | 401×9 → 429, Retry-After 60, ответы двух реплик; JSON > 64 KiB → 413 (и chunked), upload 50 MB → 413 на LB; посторонний → 404 ×6; logout → 401 | `lab7-security.txt` |
|
||||
| Сертификаты — локальный документированный bootstrap | `pki.py init` → `init_secrets.py` → `bao.py init/unseal/configure` | чистый Docker-стенд по этим шагам поднят с нуля (новые secrets/, volumes) | раздел «Запуск» |
|
||||
| Smoke, рестарт и отказ реплики через HTTPS без отключения проверки | `run_persistence.py`, `run_failover.py` | PASS (рестарт 16 сервисов, включая OpenBao + unseal, 82.1 с) / PASS ×2 (0 неудачных записей) | `persistence.txt`, `failover.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Уровень 4
|
||||
|
||||
## Эксперименты и отказы
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker) | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| mTLS API/worker→broker и →Collector с разными identity; сертификат того же CA не даёт лишних прав | `test_tls_matrix.py broker collector` | broker: worker/dispatcher входят, CN=api/intruder/lb → login refused, worker publish / dispatcher basic_get / declare → 403; Collector: api/worker/dispatcher/janitor → 200, lb/grafana/operator/intruder → 403 | `tls-matrix.json` |
|
||||
| Пароли/token hashes и ошибки не раскрывают credentials | `test_errors_and_storage_do_not_reveal_credentials`, `leak_check.py` | ошибки без эха пароля/username; в Redis только `session:<SHA-256>`; в логах/телеметрии — 0 маркеров | `lab7-security.txt`, `leak-check.json` |
|
||||
| Rehash старого хеша при входе | `test_rehash_on_login` | m=12288,t=3 (lab2–6) → после входа m=32768,t=2,p=1, повторный вход работает | `lab7-security.txt` |
|
||||
| Измеренный бюджет защиты от login DoS; rate limit общий для реплик | `login_dos.py`, `test_login_rate_limit_is_shared` | см. «Эксперименты»: атака получает 503 busy / 429, легитимные запросы — ограниченная задержка | `login-dos.json` |
|
||||
| Ротация рабочего secret и leaf без пересборки, отзыв старой identity | `lifecycle.py rotate-secret rotate-leaf` | Redis-пароль: v1→v2, обе реплики подхватили, активен только новый, 0 ошибок; leaf lb: rotate → reload → revoke → старый на новом соединении отвергнут, API закрыл 2 соединения со старым leaf, 0 ошибок | `lifecycle-rotate-*.json` |
|
||||
| Ограниченное поведение при expiry / sealed OpenBao; bootstrap trust и renewal описаны; нет постоянного «секрета для получения секретов» | `lifecycle.py token-expiry sealed`; `contracts/trust-matrix.md` «Bootstrap и эксплуатация» | TTL 20 с/max 40 с + отзыв: повторный вход сертификатом, 2393 ответов, 0 ошибок; sealed 75.6 с: работающая реплика без ошибок, перезапущенная — не ready, после unseal ready через 5.4 с | `lifecycle-token-expiry.json`, `lifecycle-sealed.json` |
|
||||
| read_only / no-new-privileges / drop caps / tmpfs; исключения stateful объяснены | `check_hardening.py --runtime` | см. выше; исключения — в `EXCEPTIONS` скрипта и в `security/THREAT_MODEL.md` (T12) | `hardening.json` |
|
||||
| Dependency/image scan с разбором применимости | pip-audit, `trivy config`, `scripts/scan.py` | pip-audit: Pillow 12.3.0 — 39 находок, применимо (worker декодирует недоверенные изображения) → 13.0.0 + `formats=(PNG, JPEG)`, повторно 0. Trivy образов (Docker): OpenSSL 3.5.1 (CRITICAL) и zlib/sqlite/libuuid в базовом `python:3.11.13-slim` — применимо (TLS каждого процесса) → `apt-get upgrade` в Dockerfile, OpenSSL 3.5.7, applicable 37 → 9 (оставшиеся без исправления и недостижимы: модули не импортируются); сторонние образы разобраны по ролям, открыто — RabbitMQ (OpenSSL 3.5.4), OpenBao (Go stdlib), HAProxy (musl/pcre2/zlib) | `security/scan-triage.md`, `evidence/scan/` |
|
||||
| Модель угроз: ≥6 угроз → контроли → отрицательные тесты | `security/THREAT_MODEL.md` | 14 угроз, у каждой — тест из списка выше | — |
|
||||
| Upload-worker ограничен по CPU/RAM/времени | `test_jobs.py limits` (lab5), compose: worker 2 CPU / 1536 MiB / 128 PID, `imaging.py` RLIMIT_AS/CPU + таймаут | PASS | `jobs-limits.txt` |
|
||||
| TLS-аудит всей заявленной матрицы, ACL и отсутствие утечек в telemetry | `test_tls_matrix.py`, `test_lab7_security.py`, `leak_check.py` | все PASS | см. выше |
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
<!-- measurements:begin -->
|
||||
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-07, commit 20afc2c + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB; lab7: openbao 0,5 CPU / 256 MiB, otel-gate 0,25 CPU / 64 MiB; у всех — лимит PID.
|
||||
|
||||
TLS/mTLS-аудит trust matrix изнутри сети (`student-tests/test_tls_matrix.py`, контейнер probe):
|
||||
|
||||
| Связь | Проверок | Пройдено |
|
||||
| --- | --- | --- |
|
||||
| LB -> API (mTLS, allowlist CN=lb) | 10 | 10 |
|
||||
| API/worker -> RabbitMQ (mTLS + EXTERNAL, CN = broker user) | 11 | 11 |
|
||||
| API/worker -> OTel Collector (mTLS via otel-gate, allowlist app CNs) | 13 | 13 |
|
||||
| API/worker -> PostgreSQL (TLS verify-full, SCRAM, roles) | 4 | 4 |
|
||||
| API -> Redis (TLS + client cert of our CA + ACL user) | 6 | 6 |
|
||||
| API/worker -> S3 (HTTPS + per-service keys by prefix) | 6 | 6 |
|
||||
| Services -> OpenBao (TLS + cert auth by CN + policy) | 7 | 7 |
|
||||
| Collector/Grafana/operator -> Prometheus (mTLS + allowed SANs) | 7 | 7 |
|
||||
| Collector/Grafana/operator -> Loki (mTLS, CA) | 4 | 4 |
|
||||
| Collector/Grafana/operator -> Tempo (mTLS, CA) | 4 | 4 |
|
||||
| Collector -> Tempo OTLP gRPC :4317 (mTLS) | 3 | 3 |
|
||||
| Operator -> Grafana (HTTPS) | 3 | 3 |
|
||||
|
||||
Минимальные права (`test_least_privilege.py`): 83 проверок, 83 пройдено (разрешённое работает, запрещённое — отказ).
|
||||
|
||||
Argon2id внутри контейнера api (`argon2_bench.py`, медиана из 7):
|
||||
|
||||
| m, KiB | t | p | hash, мс | verify, мс | входов/с на реплику при 2 параллельных | RAM на хеширование, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| 19456 | 2 | 1 | 12.2 | 12.2 | 163.9 | 38 |
|
||||
| 19456 | 3 | 1 | 18.3 | 18.5 | 108.1 | 38 |
|
||||
| 32768 (выбрано) | 2 | 1 | 22.6 | 22.2 | 90.1 | 64 |
|
||||
| 47104 | 2 | 1 | 34.2 | 34.8 | 57.5 | 92 |
|
||||
| 65536 | 2 | 1 | 49.8 | 50.4 | 39.7 | 128 |
|
||||
| 65536 | 3 | 1 | 75.3 | 75.2 | 26.6 | 128 |
|
||||
|
||||
Login DoS (`login_dos.py`, 16 потоков неверных входов, 20 с): атака 96.2 попыток/с, коды {'401': 360, '429': 1564}; легитимный клиент p95 34.6 -> 194.1 мс, p99 35.5 -> 243.6 мс, коды {'200': 93, '201': 93}.
|
||||
|
||||
Ротация пароля Redis (api) через OpenBao: {"kv_version": [5, 6], "replicas": 2, "active_passwords_during_overlap": 2, "all_replicas_picked_up_after_s": 23.9, "active_password_hashes_after": 1, "new_password_hash_matches": true}; нагрузка 37.6 с, ответы {'me:200': 431, 'send:201': 431, 'login:201': 18}, ошибок 5xx/соединения 0.
|
||||
|
||||
Ротация leaf LB + отзыв старого через CRL: {"old_serial": "70ea18ded7f86e5a", "new_serial": "22a71730433d9d42", "probe": {"valid LB identity (CN=lb)": "http_200", "foreign CA, CN=lb": "connection_closed_after_handshake", "our CA, expired, CN=lb": "connection_closed_after_handshake", "previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"}, "api_events_after_revoke": {"tls.crl_loaded": 2, "tls.revoked_connections_closed": 2, "tls.client_rejected": 4}}; нагрузка 23.1 с, ответы {'me:200': 243, 'send:201': 243, 'login:201': 11}, ошибок 5xx/соединения 0.
|
||||
|
||||
Истечение/отзыв токенов OpenBao: {"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": {"secrets.refresh_failed": 2, "secrets.login": 4}}; нагрузка 101.6 с, ответы {'me:200': 1172, 'send:201': 1172, 'login:201': 49}, ошибок 5xx/соединения 0.
|
||||
|
||||
OpenBao sealed: {"sealed_seconds": 75.6, "restarted_replica_health_while_sealed": "starting", "replicas_answering_while_sealed": 1, "restarted_replica_ready_after_unseal_s": 5.4, "api_events": {"secrets.refresh_failed": 5, "secrets.waiting": 8, "secrets.login": 1}}; нагрузка 89.2 с, ответы {'me:200': 1012, 'send:201': 1012, 'login:201': 43}, ошибок 5xx/соединения 0.
|
||||
|
||||
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
|
||||
|
||||
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 94.0 (от stop) | 45.6 (от start) |
|
||||
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 124.5 (от stop) | 45.6 (от start) |
|
||||
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 122.5 (от LOCK) | 35.1 (от COMMIT блокировки) |
|
||||
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 85.9 (от stop) | 70.6 (от start) |
|
||||
|
||||
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 115.4 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 123.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 5.4 с, backlog 0.0.
|
||||
|
||||
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '400': 0.0, '409': 0.0, '404': 0.0, '503': 1.44}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 15.6, 'max': 31.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 50418f127c336af7f2e4dc0190adf7ea: server-спан 5010.0 мс, из них `INSERT` 5001.6 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 2e774b9d5912ba768f3e9881f3c41e80, POST /users): server 47.0 мс, `argon2id hash` 32.2 мс, все спаны БД 6.9 мс. Нагрузка: до 102.1 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 102.4/с без ошибок.
|
||||
|
||||
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
|
||||
| --- | --- | --- | --- |
|
||||
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 8486 запросов, коды {'201': 8486}, доля успеха 1.0 | heap Collector 32.3 -> max 84.7 MiB (limit 200), контейнер max 103.7 MiB | send/enqueue failed в Collector: {'spans': 2048, 'log_records': 213}; пробные трассы дошли 13/13, потеряны отправленные на [] с |
|
||||
| `stop otel-collector` на 90 с | 5298 запросов, коды {'201': 5298}, доля успеха 1.0 | api 831.6 -> max 866.4 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 19456} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (5554 спанов за минуту) |
|
||||
|
||||
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||
|
||||
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1 | 0c7893e1043d | 2 | 126 | 0 | 5.453 |
|
||||
| 2 | b7f37f051ba1 | 2 | 125 | 0 | 5.313 |
|
||||
|
||||
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||
|
||||
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.54 / 1.64 | 0c7893e1043d: 5 | 129.5 | 141.0 | 11.5 | 164.8 |
|
||||
| | | | | b7f37f051ba1: 5 | 126.2 | 139.1 | 12.9 | 164.8 |
|
||||
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.13 / 1.39 | 0c7893e1043d: 5 | 128.9 | 134.9 | 6.0 | 113.5 |
|
||||
| | | | | b7f37f051ba1: 5 | 126.7 | 134.4 | 7.7 | 113.9 |
|
||||
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.64 / 2.25 | 0c7893e1043d: 10 | 125.6 | 144.1 | 18.5 | 174.8 |
|
||||
| | | | | b7f37f051ba1: 10 | 128.7 | 147.9 | 19.2 | 182.8 |
|
||||
|
||||
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 162 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||
|
||||
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 206.623 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 5.453, 5.313 с (hook ждёт остановки ~4 с); потерь нет |
|
||||
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 82.1 с до готовности; потерь нет |
|
||||
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 781 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 8703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6133fe0a-b568-4024-8bb7-4d7d32ed0433 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 60ca1ef4-a5fb-48ad-94bd-5f14f683304d ready, attempt 1, no manual action | без ручного действия |
|
||||
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 18 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready |
|
||||
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.70 s and 2.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 3d109dba-1036-482a-a6fb-8116c7894b6d attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (4 message(s)); dlq-replay kept them: ['unknown job', 'unknown job'] | сообщения сохранены |
|
||||
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.6 s after worker start, peak processing 2 of 8 (6 samples) | - |
|
||||
<!-- measurements:end -->
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
Docker-стенд: Windows 10, Docker Desktop (VM 16 CPU, 15.2 GiB), клиент тестов — Python 3.11 на том же хосте, 2 реплики API по 1 CPU / 512 MiB. Числа нативной песочницы (2 vCPU на всё) — в `evidence/native-sandbox/summary.md`, с этими не сравнивать.
|
||||
|
||||
**Argon2id — выбор параметров** (`argon2_bench.py`, внутри контейнера api, медиана из 7):
|
||||
|
||||
| m, KiB | t | p | hash, мс | verify, мс | входов/с на реплику (2 параллельно) | RAM на хеширование |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
<!-- argon2:begin -->
|
||||
| 19456 | 2 | 1 | 12.2 | 12.2 | 163.9 | 38 MiB |
|
||||
| 19456 | 3 | 1 | 18.3 | 18.5 | 108.1 | 38 MiB |
|
||||
| 32768 **(выбрано)** | 2 | 1 | 22.6 | 22.2 | 90.1 | 64 MiB |
|
||||
| 47104 | 2 | 1 | 34.2 | 34.8 | 57.5 | 92 MiB |
|
||||
| 65536 | 2 | 1 | 49.8 | 50.4 | 39.7 | 128 MiB |
|
||||
| 65536 | 3 | 1 | 75.3 | 75.2 | 26.6 | 128 MiB |
|
||||
<!-- argon2:end -->
|
||||
|
||||
Выбрано **m=32 MiB, t=2, p=1**: в 1,7 раза больше памяти, чем минимум OWASP/README (19 MiB, t=2), ~22 мс на проверку в контейнере api — незаметно для пользователя и вдвое дешевле 64 MiB (~50 мс). Бюджет реплики: не больше `ARGON2_MAX_CONCURRENCY=2` хешей одновременно (≤ 64 MiB из 512 MiB лимита памяти), ожидание слота не дольше 2 с, дальше — 503 + Retry-After. У api лимит 1 CPU, поэтому два параллельных хеша делят одно ядро: реальный потолок ≈ 1000 / 22 мс ≈ 45 проверок пароля в секунду на реплику (≈ 90/с на две; колонка «2 параллельно» — без учёта лимита CPU); всё, что выше, отсекают сначала общий лимит Redis (429), потом бюджет реплики (503), а не рост памяти или очередь. 64 MiB/t=2 удвоили бы и цену входа, и память на хеширование при том же бюджете, то есть вдвое снизили бы потолок входов. (В песочнице те же параметры стоили ~50 мс — выбор от этого не меняется.)
|
||||
|
||||
**Находка при проверке burst (песочница):** без настройки аллокатора glibc держит 32-MiB буферы Argon2 в отдельных arena каждого потока — 200 одновременных проверок раздували RSS реплики на +548 MiB (больше лимита 512 MiB → OOM-kill под атакой). `MALLOC_MMAP_THRESHOLD_=1048576` в окружении api заставляет крупные буферы идти через mmap и возвращаться ОС сразу: прирост RSS на том же burst — +37 MiB в песочнице, +74.7 MiB в Docker (ниже). Добавлено в `compose.yaml` и в угрозу T5.
|
||||
|
||||
**Login DoS** (`login_dos.py`):
|
||||
<!-- dos:begin -->
|
||||
| Фаза | Легитимных запросов | p50 / p95 / p99, мс | Коды легитимных | Попыток входа атаки | Коды атаки |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| без атаки, 20 с | 694 | 12.3 / 34.6 / 35.5 | {'200': 347, '201': 347} | — | — |
|
||||
| атака: 16 потоков неверных паролей по 40 username, 20 с | 186 | 91.1 / 194.1 / 243.6 | {'200': 93, '201': 93} | 1924 (96.2/с), p95 ответа 480.5 мс | {'401': 360, '429': 1564} |
|
||||
|
||||
Из 1924 попыток атаки хеш Argon2id посчитан только для 360 (ответ 401: по 9 на каждый из 40 username до срабатывания лимита), 1564 отсечены общим (Redis) лимитом 429 + Retry-After до хеширования, 503 busy — 0: до бюджета реплики дело не дошло, его отдельно проверяет burst ниже. Легитимные запросы обслуживались без ошибок; задержка выросла (p95 34.6 → 194.1 мс), и поэтому closed-loop клиент сделал за 20 с меньше запросов (694 → 186): атакующий клиент, стенд и легитимный клиент делят одну машину, а 16 потоков TLS-рукопожатий и Python-клиента сами занимают CPU хоста.
|
||||
|
||||
Burst в одном процессе api (бюджет без помощи rate limit): 200 одновременных проверок пароля → одновременно считалось не больше 2 хешей (лимит 2), посчитано 75, отклонено 503 + Retry-After 125, самое долгое ожидание отказа 2.05 с (= очередь 2 с); прирост RSS 74.7 MiB при лимите 512 MiB (с MALLOC_MMAP_THRESHOLD_=1048576; без него в песочнице было +548 MiB, см. угрозу T5).
|
||||
<!-- dos:end -->
|
||||
|
||||
**Жизненный цикл под нагрузкой** (`lifecycle.py`, фоновая нагрузка: вход + GET /users/me + POST message):
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
<!-- lifecycle:begin -->
|
||||
| Ротация пароля Redis-пользователя api через OpenBao | 0 ошибок; после — активен только новый пароль | KV v5 → v6 (предыдущие версии — от прошлых прогонов); паролей во время перехода 2, после — 1 (новый); обе реплики подхватили за 23.9 с (≤ SECRETS_REFRESH_SECONDS 30 с); нагрузка 37.6 с, 880 ответов, ошибок 0 | без рестартов и пересборки; потерь данных нет |
|
||||
| Ротация leaf-сертификата LB + отзыв старого (CRL) | 0 ошибок; старый leaf отвергается на новом соединении, открытые закрыты | старый leaf → connection_closed_after_handshake, новый → http_200; события API: {'tls.crl_loaded': 2, 'tls.revoked_connections_closed': 2, 'tls.client_rejected': 4}; нагрузка 23.1 с, 497 ответов, ошибок 0 | эффект отзыва ≤ CRL_RELOAD_SECONDS (5 с) после reload; потерь данных нет |
|
||||
| Токены OpenBao роли api: TTL 20 с / max 40 с + отзыв всех токенов | сервисы перелогиниваются сертификатом, 0 ошибок | события API за 100 с: {'secrets.refresh_failed': 2, 'secrets.login': 4}; нагрузка 101.6 с, 2393 ответов, ошибок 0 | без участия оператора; потерь данных нет |
|
||||
| OpenBao запечатан, одна реплика API перезапущена | работающая реплика обслуживает, перезапущенная не ready до unseal | sealed 75.6 с; перезапущенная реплика: starting (не healthy, LB её не использует); события API: {'secrets.refresh_failed': 5, 'secrets.waiting': 8, 'secrets.login': 1}; нагрузка 89.2 с, 2067 ответов, ошибок 0 | реплика ready через 5.4 с после unseal; потерь данных нет |
|
||||
<!-- lifecycle:end -->
|
||||
|
||||
**Отказы lab3–5 через HTTPS** (публичные тесты): `run_persistence.py` — рестарт PostgreSQL, Redis, S3, RabbitMQ, **OpenBao** (+ unseal hook'ом), телеметрии, приложения и LB с сохранением volumes; данные и прежний токен на месте. `run_failover.py` — остановка каждой реплики по очереди под записью.
|
||||
<!-- resilience:begin -->
|
||||
- persistence: PASS — the hook restarted postgres, redis, s3, rabbitmq, openbao, api, janitor, dispatcher, worker, lb, otel-gate, otel-collector, prometheus, loki, tempo, grafana (all api replicas; same containers, new start time); data, the attachment bytes and the old token survived; hook + готовность 82.1 с
|
||||
- failover, реплика 1: реплик до отказа 2, успешных записей 126, неудачных попыток (повторены тем же ключом) 0, первая запись после hook через 5.453 с
|
||||
- failover, реплика 2: реплик до отказа 2, успешных записей 125, неудачных попыток (повторены тем же ключом) 0, первая запись после hook через 5.313 с
|
||||
<!-- resilience:end -->
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Один хост, одна копия каждого хранилища** — отказ хоста, PostgreSQL, Redis, RabbitMQ, SeaweedFS или OpenBao не выдерживается (как в lab3–6). OpenBao sealed/недоступен: работающие реплики живут на кэше, новые не стартуют — сознательный fail closed.
|
||||
- **Доли unseal в одном файле оператора** (учебный компромисс): restart-hook распечатывает OpenBao без человека. В эксплуатации — `bao.py init --print-shares-once`, доли у разных людей, `unseal --prompt`, отзыв root token после настройки, CA офлайн.
|
||||
- **CRL проверяют** API (сам, с перечитыванием), HAProxy (lb, otel-gate — при reload), OpenBao cert auth (`bao.py sync-crl`). **Не проверяют**: RabbitMQ, Redis, PostgreSQL, Prometheus/Loki/Tempo — для них отзыв = удаление пользователя/смена пароля + короткий срок leaf (30 суток). Следующий шаг — короткоживущие leaf из OpenBao PKI с автопродлением (трек 5.2).
|
||||
- **Loki/Tempo** различают только «наш CA / не наш»: читатель и писатель не разделены; сервис приложения может подписать телеметрию чужим `service.name` (шлюз проверяет identity, но не содержимое).
|
||||
- **Пароли ролей PostgreSQL**: ротация описана (новая версия KV → `docker compose run --rm migrate`), но не демонстрировалась и даёт окно до 30 с отказов новых соединений; на 4 продемонстрирована ротация пароля Redis (без окна — два пароля одновременно).
|
||||
- **SeaweedFS 4.52 после рестарта** отдаёт чтение раньше записи: `/cluster/healthz` = 200 через ~20 с, но мастер ещё ~15 с отвечает «master is warming up» на `/dir/assign`, и загрузки получают 503. Флага, сокращающего прогрев, нет; healthcheck S3 переведён на `/dir/assign` (готовность к записи), `test_consistency` ждёт первую успешную загрузку (в Docker — 14.7 с после того, как старый файл снова читается; в песочнице было 16 с).
|
||||
- **Исключения hardening**: RabbitMQ с записываемым корнем; telemetry-init — root + CAP_CHOWN на время chown (идемпотентно: chown только для volume, который ещё не принадлежит 10001). SeaweedFS после Docker-прогона работает от uid 1000 (`seaweed` образа), исключение uid 0 снято.
|
||||
- **Restart hook на настоящем Docker** (в песочнице не воспроизводилось): (1) `docker compose restart` восьми хранилищ ждёт самые медленные (PostgreSQL, RabbitMQ), а запечатанный OpenBao за это время проваливает 3 healthcheck, становится `unhealthy`, и `up --wait` сразу падает, хотя через секунду OpenBao уже распечатан; (2) параллельный `restart otel-gate otel-collector …` — Collector не может войти в network namespace ещё остановленного gate. Исправлено порядком в `scripts/restart.cmd`/`.sh`: OpenBao перезапускается отдельно и распечатывается сразу (внутри `start_period`), otel-gate — отдельно перед Collector; набор перезапускаемых сервисов тот же.
|
||||
- **Нестабильные проверки, найденные в Docker** (ожидания не менялись, исправлено ожидание готовности): Loki после старта ~15 с отвечает `/ready` → 503 «Ingester not ready» — положительная проверка mTLS оператора к Loki/Tempo в `tls_probe.py` ждёт готовности до 30 с, только если mTLS пройден (ответ 503), итог по-прежнему должен быть 200; в `test_telemetry.py` трасса считалась полной по span'ам worker, а batch API/dispatcher выгружается независимо (до 5 с) и иногда приходил в Tempo позже (2 FAIL из 4) — условие ожидания теперь требует span'ы всех процессов, проверки те же (4 из 4 PASS). `lifecycle.py token-expiry`: после `revoke-service-tokens` отзыв замечает то обращение к OpenBao, которое случится раньше, — чтение KV (`secrets.refresh_failed`) или продление токена; во втором случае API сразу входил заново, но писал общее с max TTL событие `secrets.relogin`, и тест (ждал только refresh_failed) падал при работающей функции. Теперь неудачный renew — отдельное событие `secrets.renew_failed` (WARNING, `secretstore.py`), тест принимает любой из двух путей обнаружения и по-прежнему требует повторный вход и 0 ошибок. В `upload_memory.py` счётчик загрузок по репликам сравнивал X-Instance-Id с ID контейнера (в lab7 это alias) и всегда давал 0 — сопоставление через `instance_aliases()`.
|
||||
- **Память SeaweedFS 4.52**: ~240 MiB в простое; в Docker-прогоне `upload_memory.py` (20 параллельных загрузок по ~10 MiB) довёл процесс до лимита 512 MiB, ядро убило его (SIGKILL, рестарт по политике), половина загрузок получила 503. Исправлено без повышения лимита: `GOMEMLIMIT=400MiB` (GC Go ориентируется на лимит) и `-s3.concurrentUploadLimitMB=128` (сверх 128 MiB тел одновременно шлюз ставит загрузки в очередь). После — все 20 загрузок 202, рестартов S3 нет. Цена — задержка загрузок при длительном потоке крупных файлов (T14).
|
||||
- **`localhost` на хосте Windows** резолвится сначала в `::1`, а порты опубликованы только на `127.0.0.1` (IPv6-loopback не публикуем намеренно): каждое новое TCP-соединение с `https://localhost:8443` ждёт ~2 с отказа по IPv6 (замер: localhost 2031–2051 мс, 127.0.0.1 — 0 мс). Первый Docker-прогон из-за этого давал в 40 раз меньше запросов нагрузки. Скрипты `student-tests` по умолчанию ходят на `https://127.0.0.1:<порт>` (IP есть в SAN сертификатов ingress и админ-интерфейсов, имя проверяется так же строго); `https://localhost:8443` работает, только медленнее на новых соединениях.
|
||||
- **OpenBao file storage** в 2.6.2 помечен deprecated (в логе: «use bao operator migrate … by v2.7.0»). Для учебного стенда на одном хосте оставлен; перед обновлением до 2.7 — `bao operator migrate` на integrated storage (raft).
|
||||
- **Rate limit на username** позволяет заблокировать вход конкретному пользователю на окно (60 с) — плата за защиту от подбора; лимит регистрации при недоступном Redis пропускается (защищает бюджет Argon2id).
|
||||
- **Docker-прогон после песочницы** (настоящие OpenBao 2.6.2, HAProxy 3.2, RabbitMQ 4.1.4, PostgreSQL 17, Redis 7.4, Loki/Tempo/Grafana с TLS): прошли все 46 шагов. Исправлено по его итогам (подробно — пункты выше): SeaweedFS 4.52 не мог писать в `/data` от root без capabilities → запуск от uid 1000; `telemetry-init` падал на повторном `up` → идемпотентный chown; у Prometheus не было tmpfs `/tmp` для `promtool test rules`; память SeaweedFS под 20 параллельными загрузками; порядок restart hook; `localhost` → `127.0.0.1` в клиентах тестов; ожидания готовности в трёх тестах; отдельное событие `secrets.renew_failed`; по скану образов — `apt-get upgrade` в `Dockerfile` (OpenSSL 3.5.1 → 3.5.7), после чего весь прогон повторён.
|
||||
- **Открыто по скану** (`security/scan-triage.md`): OpenSSL 3.5.4 в `rabbitmq:4.1.4-alpine` (порт только в internal-сети `data`, вход по mTLS), Go stdlib/x/net в OpenBao 2.6.2, musl/pcre2/zlib в HAProxy 3.2.25 — закрываются обновлением тегов до патч-релизов с повторным прогоном; в этой работе версии сторонних сервисов не менялись. Конфигурации OpenBao, HAProxy, RabbitMQ (EXTERNAL), PostgreSQL (ключ в /tmp/pgtls), TLS Collector → Loki/Tempo/Prometheus, Grafana → источники и внутренний gRPC Tempo заработали без изменений.
|
||||
|
||||
## Ответы на вопросы защиты (кратко)
|
||||
|
||||
- **Какая часть mTLS проверяет шифрование, какая идентичность и где разрешение на действие?** Шифрование и целостность — TLS-сессия (ECDHE). Идентичность сервера — проверка цепочки до нашего CA и имени (verify-full / check_hostname / verifyhost). Идентичность клиента — его сертификат (цепочка, срок, CRL). Разрешение — отдельный шаг после рукопожатия: API — allowlist CN=lb; otel-gate — CN ∈ приложений; RabbitMQ — пользователь = CN и его permissions; OpenBao — роль cert auth = CN → policy; Prometheus — `client_allowed_sans`; PostgreSQL/Redis/S3 — по роли/ACL/ключу. Пользовательская сессия (Bearer) — ещё выше, внутри API.
|
||||
- **Почему доверенный CA ещё не означает, что любой его клиент может быть LB?** Один CA выпускает сертификаты всем сервисам; «подписан нашим CA» говорит только «это кто-то из наших». Если пускать любого, скомпрометированный worker стал бы балансировщиком (обход лимитов, подделка XFF). Поэтому API проверяет CN=lb — в тесте сертификаты worker, api и intruder нашего CA отвергаются.
|
||||
- **Где хранится первый секрет для доступа к OpenBao и кто имеет право unseal?** Первый секрет сервиса — его leaf-сертификат и ключ (`secrets/pki/<identity>`, смонтирован только в этот сервис, 30 суток, отзывается CRL). SecretID/токенов в env нет. Unseal — оператор: 3 доли, порог 2 (на стенде в `secrets/operator/openbao-init.json`, 0600, вне Git и контейнеров; в эксплуатации — у разных людей).
|
||||
- **Что происходит при истечении credentials, остановке OpenBao и рестарте API?** Токен OpenBao продлевается и при исчерпании max TTL заменяется новым входом по сертификату (без влияния на запросы). Истёкший leaf — отказ рукопожатия, нужен `pki.py rotate`. OpenBao остановлен/sealed — работающие процессы обслуживают запросы с секретами в памяти, обновление ротаций откладывается; рестартованная реплика не становится ready, LB её не использует; после unseal — восстанавливается сама. Рестарт API при работающем OpenBao — обычный: вход сертификатом, секреты за доли секунды.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202.
|
||||
|
||||
Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого
|
||||
объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение:
|
||||
|
||||
1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503.
|
||||
2. PUT исходника в staging/<hex> — упало: удаляем объект (PUT мог пройти)
|
||||
и строку, клиенту 503. Если и это не вышло —
|
||||
uploading-строку уберёт janitor после grace.
|
||||
3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT
|
||||
— упало/неизвестно: клиенту 503, строка
|
||||
остаётся uploading -> janitor удалит объект
|
||||
и строку. Если COMMIT всё же прошёл, задание
|
||||
есть и будет обработано (повтор клиента даст
|
||||
второе вложение — в контракте нет ключа).
|
||||
4. 202 Attachment(status=queued) — только после шага 3.
|
||||
|
||||
Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный
|
||||
брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно.
|
||||
"""
|
||||
import logging
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import db
|
||||
import settings
|
||||
import storage
|
||||
from errors import unavailable
|
||||
from uploads import ImageInfo, Upload
|
||||
|
||||
log = logging.getLogger("messenger.attachments")
|
||||
|
||||
|
||||
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
|
||||
attachment_id = uuid4()
|
||||
key = storage.new_key(uuid4().hex)
|
||||
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
|
||||
upload.size, upload.sha256, image.width, image.height, key)
|
||||
try:
|
||||
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
|
||||
except storage.StorageUnavailable:
|
||||
compensate(attachment_id, key)
|
||||
raise unavailable("File storage is temporarily unavailable") from None
|
||||
row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID)
|
||||
if row is None:
|
||||
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
|
||||
log.warning("Upload %s was reclaimed before it was queued", attachment_id)
|
||||
raise unavailable()
|
||||
log.info("Attachment %s accepted into staging and queued", attachment_id)
|
||||
return {**row, "variants": []}
|
||||
|
||||
|
||||
def compensate(attachment_id: UUID, key: str) -> None:
|
||||
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
|
||||
try:
|
||||
storage.delete(key, quick=True)
|
||||
db.delete_upload_intent(attachment_id)
|
||||
log.info("Upload %s rolled back after S3 failure", attachment_id)
|
||||
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
|
||||
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
|
||||
|
||||
|
||||
def public(row: dict) -> dict:
|
||||
"""Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей)."""
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["state"],
|
||||
"variants": [
|
||||
{key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
|
||||
for v in row.get("variants", [])
|
||||
],
|
||||
"error_code": row["error_code"] if row["state"] == "failed" else None,
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Пароли (Argon2id), сессии и ограничение частоты входа (lab7).
|
||||
|
||||
Пароли: argon2-cffi, Argon2id, параметры из settings (не ниже m=19 MiB, t=2, p=1; выбор — по замеру,
|
||||
REPORT.md). В БД хранится строка $argon2id$v=19$m=...,t=...,p=...$<salt>$<hash>: соль и параметры
|
||||
вместе с хешем. При успешном входе hash со старыми параметрами пересчитывается (rehash) и сохраняется.
|
||||
|
||||
Бюджет CPU/RAM: Argon2id намеренно дорог, поэтому поток запросов входа/регистрации — готовый DoS.
|
||||
- одновременно в реплике не больше ARGON2_MAX_CONCURRENCY хешей (память N x m, CPU N ядер);
|
||||
ожидание слота не дольше ARGON2_QUEUE_SECONDS, иначе 503 + Retry-After (нет растущей очереди);
|
||||
- ДО хеширования — общий для всех реплик счётчик в Redis (фиксированное окно): попыток на username
|
||||
и на IP клиента; превышение -> 429 + Retry-After, хеш не считается вовсе.
|
||||
- несуществующий пользователь проверяется против DUMMY_HASH с теми же параметрами: время ответа
|
||||
не выдаёт, есть ли такой username.
|
||||
|
||||
Сессии: случайный токен 256 бит; в Redis хранится только SHA-256 токена (session:<hex>) -> user_id
|
||||
с абсолютным временем истечения. Redis — TLS + клиентский сертификат + ACL-пользователь api, у которого
|
||||
есть доступ только к ключам session:* и ratelimit:* и только к нужным командам; пароль ACL-пользователя
|
||||
берётся из OpenBao на каждое новое соединение (ротация без рестарта).
|
||||
"""
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
import ssl
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from argon2 import PasswordHasher, Type
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.credentials import CredentialProvider
|
||||
from redis.retry import Retry
|
||||
|
||||
import settings
|
||||
import telemetry
|
||||
from errors import BASIC_CHALLENGE, busy, rate_limited, unauthorized, unavailable
|
||||
from secretstore import store
|
||||
|
||||
log = logging.getLogger("messenger.auth")
|
||||
|
||||
hasher = PasswordHasher(time_cost=settings.ARGON2_ITERATIONS, memory_cost=settings.ARGON2_MEMORY_KIB,
|
||||
parallelism=settings.ARGON2_PARALLELISM, hash_len=32, salt_len=16, type=Type.ID)
|
||||
|
||||
DUMMY_HASH = hasher.hash("there-is-no-such-user")
|
||||
|
||||
PASSWORD_MIN_LENGTH = 12
|
||||
PASSWORD_MAX_LENGTH = 128
|
||||
|
||||
_hash_slots = threading.BoundedSemaphore(settings.ARGON2_MAX_CONCURRENCY)
|
||||
HASH_REJECTED = telemetry.meter.create_counter(
|
||||
"messenger.auth.hash_rejected", unit="{request}",
|
||||
description="Password hashing requests rejected because the per-replica Argon2id budget was exhausted")
|
||||
RATE_LIMITED = telemetry.meter.create_counter(
|
||||
"messenger.auth.rate_limited", unit="{request}", description="Login/registration attempts answered with 429")
|
||||
REHASHED = telemetry.meter.create_counter(
|
||||
"messenger.auth.rehashed", unit="{user}", description="Password hashes upgraded to current Argon2id parameters")
|
||||
|
||||
|
||||
class _HashSlot:
|
||||
"""Слот бюджета хеширования; занят — ждём не дольше ARGON2_QUEUE_SECONDS, иначе 503."""
|
||||
|
||||
def __enter__(self):
|
||||
if not _hash_slots.acquire(timeout=settings.ARGON2_QUEUE_SECONDS):
|
||||
HASH_REJECTED.add(1)
|
||||
log.warning("Argon2id budget exhausted", extra={"event": "auth.hash_budget_exhausted"})
|
||||
raise busy(1)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
_hash_slots.release()
|
||||
return False
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
# Argon2id нагружает CPU обработчика (а не БД): в трассе это отдельный спан.
|
||||
with _HashSlot(), telemetry.internal_span("argon2id hash"):
|
||||
return hasher.hash(password)
|
||||
|
||||
|
||||
def password_matches(password_hash: str | None, password: str) -> bool:
|
||||
with _HashSlot(), telemetry.internal_span("argon2id verify"):
|
||||
try:
|
||||
hasher.verify(password_hash or DUMMY_HASH, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
return password_hash is not None
|
||||
|
||||
|
||||
def needs_rehash(password_hash: str) -> bool:
|
||||
try:
|
||||
return hasher.check_needs_rehash(password_hash)
|
||||
except InvalidHashError:
|
||||
return False
|
||||
|
||||
|
||||
def rehash(password: str) -> str:
|
||||
return hash_password(password)
|
||||
|
||||
|
||||
def hash_parameters(password_hash: str) -> dict:
|
||||
"""Параметры из строки хеша (для тестов/отчёта): {"type": "argon2id", "m": ..., "t": ..., "p": ...}."""
|
||||
parts = password_hash.split("$")
|
||||
params = dict(item.split("=") for item in parts[3].split(","))
|
||||
return {"type": parts[1], "m": int(params["m"]), "t": int(params["t"]), "p": int(params["p"]),
|
||||
"salt_bytes": len(base64.b64decode(parts[4] + "=" * (-len(parts[4]) % 4)))}
|
||||
|
||||
|
||||
def parse_basic(header: str | None) -> tuple[str, str] | None:
|
||||
"""'Basic base64(username:password)' -> (username, password) или None.
|
||||
|
||||
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
|
||||
"""
|
||||
scheme, _, value = (header or "").strip().partition(" ")
|
||||
if scheme.lower() != "basic" or not value or len(value) > 1024:
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
return None
|
||||
username, colon, password = decoded.partition(":")
|
||||
if not colon or not username or not username.isascii():
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: str | None) -> str | None:
|
||||
scheme, _, value = (header or "").strip().partition(" ")
|
||||
value = value.strip()
|
||||
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
KEY_PREFIX = "session:"
|
||||
RATE_PREFIX = "ratelimit:"
|
||||
# Атрибуты спанов Redis: система и операция. Ключ (session:<SHA-256 токена>) и значения не пишутся.
|
||||
REDIS_ATTRS = {"db.system.name": "redis", "server.address": "redis"}
|
||||
|
||||
|
||||
class _RedisCredentials(CredentialProvider):
|
||||
"""ACL-пользователь и его пароль — текущая версия из OpenBao на каждое новое соединение."""
|
||||
|
||||
def get_credentials(self):
|
||||
return settings.REDIS_USER, store.get("redis_password")
|
||||
|
||||
|
||||
_redis = redis.Redis(
|
||||
connection_pool=redis.BlockingConnectionPool(
|
||||
connection_class=redis.SSLConnection,
|
||||
host=settings.REDIS_HOST,
|
||||
port=settings.REDIS_PORT,
|
||||
db=0,
|
||||
credential_provider=_RedisCredentials(),
|
||||
ssl_ca_certs=settings.TLS_CA_FILE,
|
||||
ssl_certfile=settings.TLS_CERT_FILE,
|
||||
ssl_keyfile=settings.TLS_KEY_FILE,
|
||||
ssl_cert_reqs=ssl.CERT_REQUIRED,
|
||||
ssl_check_hostname=True,
|
||||
ssl_min_version=ssl.TLSVersion.TLSv1_2,
|
||||
max_connections=settings.REDIS_POOL_MAX,
|
||||
timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
decode_responses=True,
|
||||
lib_name=None, # без CLIENT SETINFO: ACL-пользователю api эта команда не нужна и не выдана
|
||||
lib_version=None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
user_id: UUID
|
||||
key: str
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _redis_failed(exc: Exception, where: str):
|
||||
log.warning("Redis is unavailable (%s): %s", where, type(exc).__name__,
|
||||
extra={"event": "redis.unavailable", "error.type": type(exc).__name__})
|
||||
return unavailable()
|
||||
|
||||
|
||||
def check_rate(kind: str, subject: str, limit: int, fail_open: bool = False) -> None:
|
||||
"""Общий для реплик счётчик попыток (фиксированное окно). Превышение -> 429 + Retry-After.
|
||||
|
||||
В ключе — SHA-256 субъекта (username/IP), а не сам субъект: Redis не хранит имена и адреса.
|
||||
fail_open: при недоступном Redis пропустить (регистрация с lab2 работает без Redis; от перегрузки
|
||||
её по-прежнему защищает бюджет Argon2id реплики). Вход без Redis невозможен и так (сессии в Redis).
|
||||
"""
|
||||
key = RATE_PREFIX + kind + ":" + hashlib.sha256(subject.encode("utf-8")).hexdigest()[:32]
|
||||
window_ms = settings.LOGIN_RATE_WINDOW_SECONDS * 1000
|
||||
try:
|
||||
with telemetry.client_span("redis INCR", REDIS_ATTRS | {"db.operation.name": "INCR"}):
|
||||
pipe = _redis.pipeline(transaction=True)
|
||||
pipe.incr(key)
|
||||
pipe.pexpire(key, window_ms, nx=True)
|
||||
pipe.pttl(key)
|
||||
count, _, ttl_ms = pipe.execute()
|
||||
except redis.RedisError as exc:
|
||||
if fail_open:
|
||||
log.warning("Rate limit skipped, Redis is unavailable: %s", type(exc).__name__,
|
||||
extra={"event": "auth.rate_limit_skipped", "error.type": type(exc).__name__})
|
||||
return
|
||||
raise _redis_failed(exc, "rate limit") from None
|
||||
if count > limit:
|
||||
RATE_LIMITED.add(1, {"messenger.rate_limit": kind})
|
||||
log.info("Rate limited: %s", kind, extra={"event": "auth.rate_limited", "messenger.rate_limit": kind})
|
||||
raise rate_limited(-(-max(ttl_ms, 1000) // 1000))
|
||||
|
||||
|
||||
def create_session(user_id: UUID) -> tuple[str, datetime]:
|
||||
"""Новая сессия: случайный токен и абсолютное время истечения."""
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
|
||||
try:
|
||||
with telemetry.client_span("redis SET", REDIS_ATTRS | {"db.operation.name": "SET"}):
|
||||
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
|
||||
except redis.RedisError as exc:
|
||||
raise _redis_failed(exc, "login") from None
|
||||
return token, expires_at
|
||||
|
||||
|
||||
def check_bearer(header: str | None) -> Session:
|
||||
token = parse_bearer(header)
|
||||
if token is None:
|
||||
raise unauthorized("Bearer token is required")
|
||||
key = session_key(token)
|
||||
try:
|
||||
with telemetry.client_span("redis GET", REDIS_ATTRS | {"db.operation.name": "GET"}):
|
||||
user_id = _redis.get(key)
|
||||
except redis.RedisError as exc:
|
||||
raise _redis_failed(exc, "session") from None
|
||||
if user_id is None:
|
||||
raise unauthorized("Session is invalid or expired")
|
||||
return Session(UUID(user_id), key)
|
||||
|
||||
|
||||
def delete_session(session: Session) -> None:
|
||||
try:
|
||||
with telemetry.client_span("redis DEL", REDIS_ATTRS | {"db.operation.name": "DEL"}):
|
||||
_redis.delete(session.key)
|
||||
except redis.RedisError as exc:
|
||||
raise _redis_failed(exc, "logout") from None
|
||||
|
||||
|
||||
def ping_redis() -> None:
|
||||
try:
|
||||
_redis.ping()
|
||||
except redis.RedisError:
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def login_failed():
|
||||
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
|
||||
|
||||
|
||||
def close() -> None:
|
||||
_redis.close()
|
||||
_redis.connection_pool.disconnect()
|
||||
@@ -0,0 +1,133 @@
|
||||
"""RabbitMQ: подключение (lab7: AMQPS + mTLS, EXTERNAL), проверка топологии, публикация с publisher confirms.
|
||||
|
||||
Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует
|
||||
scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure:
|
||||
они только проверяют её пассивным declare и не могут создать/удалить очередь.
|
||||
|
||||
exchange attachments (direct, durable)
|
||||
--attachment.process.v1--> queue attachments.process.v1 (quorum, durable)
|
||||
x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once),
|
||||
x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря)
|
||||
exchange attachments.dlx (direct, durable)
|
||||
--attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py
|
||||
через basic_get + nack не расходует delivery-limit)
|
||||
|
||||
Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до
|
||||
подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pika
|
||||
from pika.exceptions import AMQPError
|
||||
|
||||
import pipeline
|
||||
import settings
|
||||
import tlsutil
|
||||
|
||||
log = logging.getLogger("messenger.broker")
|
||||
logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал
|
||||
|
||||
BrokerError = (AMQPError, OSError)
|
||||
|
||||
|
||||
def parameters(url: str | None = None) -> pika.URLParameters:
|
||||
"""lab7: amqps://rabbitmq:5671/messenger — TLS с проверкой CA и имени, клиентский сертификат сервиса,
|
||||
вход механизмом EXTERNAL (пользователь брокера = CN сертификата: worker / dispatcher). Пароля нет."""
|
||||
params = pika.URLParameters(url or settings.BROKER_URL)
|
||||
host = params.host
|
||||
params.ssl_options = pika.SSLOptions(tlsutil.client_context(), server_hostname=host)
|
||||
params.credentials = pika.credentials.ExternalCredentials()
|
||||
params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS
|
||||
params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно
|
||||
params.socket_timeout = 5
|
||||
params.connection_attempts = 1
|
||||
params.client_properties = {"connection_name": settings.INSTANCE_ID}
|
||||
return params
|
||||
|
||||
|
||||
def connect(url: str | None = None) -> pika.BlockingConnection:
|
||||
return pika.BlockingConnection(parameters(url))
|
||||
|
||||
|
||||
def check_topology(channel, need=("exchange", "queue", "dlq")) -> dict:
|
||||
"""Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях.
|
||||
|
||||
lab7: каждая проверка — на отдельном временном канале. RabbitMQ 4.2+ отвечает 403 на пассивный declare
|
||||
ресурса, на который у пользователя нет НИКАКИХ прав (worker не видит DLQ и exchange, dispatcher — основную
|
||||
очередь): такой ресурс помечается None, а рабочий канал вызывающего не закрывается. Прав не расширяем.
|
||||
Если не виден ни один ресурс — это ошибка (нет топологии/прав вообще).
|
||||
"""
|
||||
connection = channel.connection
|
||||
|
||||
def passive(declare):
|
||||
temp = connection.channel()
|
||||
try:
|
||||
return declare(temp)
|
||||
except pika.exceptions.ChannelClosedByBroker as exc:
|
||||
if exc.reply_code == 403:
|
||||
return None
|
||||
raise
|
||||
finally:
|
||||
if temp.is_open:
|
||||
temp.close()
|
||||
|
||||
exchange = passive(lambda ch: ch.exchange_declare(pipeline.EXCHANGE, passive=True)) if "exchange" in need else None
|
||||
main = passive(lambda ch: ch.queue_declare(pipeline.QUEUE, passive=True).method) if "queue" in need else None
|
||||
dlq = passive(lambda ch: ch.queue_declare(pipeline.DLQ, passive=True).method) if "dlq" in need else None
|
||||
if all(item is None for item in (exchange, main, dlq)):
|
||||
raise pika.exceptions.ChannelClosedByBroker(403, "no broker resource is visible to this identity")
|
||||
return {"queue": main.message_count if main else None, "consumers": main.consumer_count if main else None,
|
||||
"dlq": dlq.message_count if dlq else None}
|
||||
|
||||
|
||||
def properties(message: dict) -> pika.BasicProperties:
|
||||
# W3C Trace Context в native headers AMQP (основной carrier для worker) и в теле (поле схемы).
|
||||
headers = {key: message[key] for key in ("traceparent", "tracestate") if key in message}
|
||||
return pika.BasicProperties(
|
||||
delivery_mode=pika.DeliveryMode.Persistent,
|
||||
content_type="application/json",
|
||||
message_id=f"{message['job_id']}:{message['attempt']}",
|
||||
type=message["type"],
|
||||
headers=headers or None,
|
||||
)
|
||||
|
||||
|
||||
class Publisher:
|
||||
"""Канал в режиме confirm: basic_publish возвращает управление только после ack брокера.
|
||||
|
||||
mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError.
|
||||
Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях
|
||||
строка outbox остаётся неопубликованной и будет отправлена позже.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
|
||||
def open(self):
|
||||
if self.channel is None or self.channel.is_closed or self.connection.is_closed:
|
||||
self.close()
|
||||
self.connection = connect()
|
||||
self.channel = self.connection.channel()
|
||||
self.channel.confirm_delivery()
|
||||
check_topology(self.channel)
|
||||
log.info("connected to broker, publisher confirms enabled", extra={"event": "broker.connected"})
|
||||
return self.channel
|
||||
|
||||
def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY):
|
||||
body = json.dumps(message, separators=(",", ":")).encode("utf-8")
|
||||
self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True)
|
||||
|
||||
def pump(self):
|
||||
if self.connection is not None and self.connection.is_open:
|
||||
self.connection.process_data_events(time_limit=0) # heartbeat в паузах
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
if self.connection is not None and self.connection.is_open:
|
||||
self.connection.close()
|
||||
except BrokerError:
|
||||
pass
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
+731
@@ -0,0 +1,731 @@
|
||||
name: messenger-lab7
|
||||
|
||||
# Lab7: безопасность. Порядок первого запуска (REPORT.md, «Запуск из чистого clone»):
|
||||
# pki.py init -> init_secrets.py -> up openbao -> bao.py init/unseal/configure -> up --build --wait
|
||||
# В репозитории нет ни одного секрета: secrets/ (CA, ключи, пароли) создаёт оператор локально, .gitignore.
|
||||
#
|
||||
# Сети (internal = без выхода наружу и без публикации портов):
|
||||
# ingress — только lb; публикуется 127.0.0.1:8443 (HTTPS) и 127.0.0.1:8404 (статистика, mTLS оператора)
|
||||
# frontend — internal: lb <-> api (mTLS, у API только CN=lb)
|
||||
# data — internal: приложение <-> PostgreSQL / Redis / S3 / RabbitMQ (все TLS)
|
||||
# secrets — internal: сервисы <-> OpenBao (TLS + cert auth)
|
||||
# telemetry — internal: приложение -> otel-gate (mTLS) -> Collector -> Prometheus/Loki/Tempo (mTLS) <- Grafana
|
||||
# ops — только для публикации админ-интерфейсов на 127.0.0.1 (OpenBao, Prometheus, Loki, Tempo, Grafana)
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
# Общие меры для контейнеров: только чтение корня, без capabilities, без повышения прав, не root
|
||||
# (USER 10001 в Dockerfile приложения; у остальных — user образа). Пишут только в tmpfs /tmp и свои volumes.
|
||||
# Число процессов ограничено в deploy.resources.limits.pids каждого сервиса.
|
||||
x-hardening: &hardening
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
|
||||
# TLS-файлы процессов приложения (смонтированы каталогами: ротация leaf видна без пересоздания контейнера)
|
||||
# и адрес OpenBao. BAO_ROLE = CN сертификата сервиса = роль cert auth = policy.
|
||||
x-tls-env: &tls-env
|
||||
TLS_CA_FILE: /run/tls-ca/ca.crt
|
||||
TLS_CRL_FILE: /run/tls-ca/crl.pem
|
||||
TLS_CERT_FILE: /run/tls/tls.crt
|
||||
TLS_KEY_FILE: /run/tls/tls.key
|
||||
BAO_ADDR: https://openbao:8200
|
||||
SECRETS_REFRESH_SECONDS: ${SECRETS_REFRESH_SECONDS:-30}
|
||||
|
||||
x-db-env: &db-env
|
||||
DB_HOST: postgres
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: messenger
|
||||
DB_SSLMODE: verify-full
|
||||
|
||||
x-pipeline-env: &pipeline-env
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||||
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
||||
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
# Учебный стенд: точки внедрения сбоев (срабатывают только по файлу в volume faults worker/dispatcher).
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-1}
|
||||
FAULTS_DIR: /faults
|
||||
|
||||
# lab6 -> lab7: OTLP/HTTP по mTLS к otel-gate (имя otel-collector), сертификат сервиса = его identity.
|
||||
x-otel-env: &otel-env
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls-ca/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/tls.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/tls.key
|
||||
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||||
OTEL_METRIC_EXPORT_INTERVAL: "10000"
|
||||
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
|
||||
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
|
||||
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
|
||||
x-otel-depends: &otel-depends
|
||||
condition: service_started
|
||||
required: false
|
||||
|
||||
x-alive-check: &alive-check
|
||||
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
|
||||
x-bao-ready: &bao-ready
|
||||
condition: service_healthy
|
||||
|
||||
services:
|
||||
# ---------------- секреты ----------------
|
||||
# OpenBao 2.6.2, НЕ dev-mode: file storage в volume, TLS-listener, после старта sealed до unseal оператором.
|
||||
# healthy = unsealed (bao status: 0 — unsealed, 2 — sealed). 127.0.0.1:8200 — только для оператора.
|
||||
openbao:
|
||||
image: openbao/openbao:2.6.2
|
||||
command: ["server"] # entrypoint образа: bao server -config=/openbao/config
|
||||
user: "100" # пользователь openbao образа, не root
|
||||
environment:
|
||||
SKIP_SETCAP: "true"
|
||||
SKIP_CHOWN: "true"
|
||||
BAO_ADDR: https://127.0.0.1:8200
|
||||
BAO_CACERT: /run/tls-ca/ca.crt
|
||||
volumes:
|
||||
- ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro
|
||||
- ./secrets/pki/openbao:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- openbaodata:/openbao/file
|
||||
ports:
|
||||
- "127.0.0.1:8200:8200"
|
||||
networks: [secrets, ops]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD", "bao", "status", "-address=https://127.0.0.1:8200", "-ca-cert=/run/tls-ca/ca.crt"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||||
|
||||
# ---------------- хранилища ----------------
|
||||
postgres:
|
||||
image: postgres:17.11-alpine
|
||||
user: "70:70" # postgres образа alpine; entrypoint не делает chown (не нужен CAP_CHOWN)
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
# Ключ TLS копируется в tmpfs с правами 0600 (PostgreSQL отказывается от ключа, доступного группе/всем,
|
||||
# а права bind mount на Windows не управляются). Только hostssl + SCRAM (infra/postgres/pg_hba.conf).
|
||||
entrypoint: ["sh", "-c", "install -d -m 0700 /tmp/pgtls && install -m 0600 /run/tls/tls.key /tmp/pgtls/tls.key && install -m 0644 /run/tls/tls.crt /tmp/pgtls/tls.crt && exec docker-entrypoint.sh \"$$@\"", "--"]
|
||||
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on",
|
||||
"-c", "ssl=on", "-c", "ssl_cert_file=/tmp/pgtls/tls.crt", "-c", "ssl_key_file=/tmp/pgtls/tls.key",
|
||||
"-c", "ssl_min_protocol_version=TLSv1.2", "-c", "password_encryption=scram-sha-256",
|
||||
"-c", "hba_file=/etc/postgresql/pg_hba.conf", "-c", "log_connections=on"]
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||||
- ./secrets/pki/postgres:/run/tls:ro
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [data]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:size=16m,mode=1777", "/var/run/postgresql:size=1m,uid=70,gid=70,mode=0775"]
|
||||
shm_size: 128m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] # unix-socket внутри контейнера
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
redis:
|
||||
image: redis:7.4.11-alpine
|
||||
user: "999:1000" # redis образа; без root и без chown
|
||||
# Стартовый ACL копируется в volume один раз; дальше ACL живёт в volume (ACL SAVE после ротации).
|
||||
command: ["sh", "-c", "[ -f /data/users.acl ] || cp /run/redis-acl/users.acl /data/users.acl; exec redis-server /usr/local/etc/redis/redis.conf"]
|
||||
volumes:
|
||||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
- ./secrets/redis:/run/redis-acl:ro
|
||||
- ./secrets/pki/redis:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- redisdata:/data
|
||||
networks: [data]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||||
healthcheck:
|
||||
# Жив и отвечает по TLS; без AUTH сервер отвечает NOAUTH — это и есть «работает и закрыт».
|
||||
test: ["CMD-SHELL", "redis-cli --tls --cacert /run/tls-ca/ca.crt --cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis ping 2>&1 | grep -qE 'NOAUTH|PONG'"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||||
|
||||
# SeaweedFS 4.52: master/volume/filer слушают только 127.0.0.1 (-ip.bind), наружу — только S3 HTTPS :8333
|
||||
# (-s3.ip.bind=0.0.0.0) и gRPC шлюза :18333 под mTLS CN=s3 (security.toml). Iceberg/Lance/IAM API выключены.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.52
|
||||
entrypoint: ["/usr/bin/weed"]
|
||||
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
|
||||
"-master.volumeSizeLimitMB=1024", "-volume.max=0", "-master.telemetry=false",
|
||||
"-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0",
|
||||
"-s3.cert.file=/run/tls/tls.crt", "-s3.key.file=/run/tls/tls.key",
|
||||
"-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.iam=false",
|
||||
"-s3.concurrentUploadLimitMB=128",
|
||||
"-s3.config=/run/secrets/s3_config"]
|
||||
# Docker-прогон: ~240 MiB в простое, 20 параллельных загрузок по ~10 MiB доводили процесс до лимита 512 MiB
|
||||
# и SIGKILL (cgroup OOM) — API получал 503. Go GC теперь ориентируется на лимит, а сверх 128 MiB тел
|
||||
# загрузок одновременно S3-шлюз ставит запросы в очередь вместо роста памяти.
|
||||
environment:
|
||||
GOMEMLIMIT: 400MiB
|
||||
secrets: [s3_config]
|
||||
volumes:
|
||||
- ./infra/seaweedfs/security.toml:/etc/seaweedfs/security.toml:ro
|
||||
- ./secrets/pki/s3:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- s3data:/data
|
||||
networks: [data]
|
||||
# Пользователь seaweed образа 4.52 (uid/gid 1000, владелец /data). Entrypoint образа делает chown + su-exec
|
||||
# от root; мы его не используем (entrypoint = weed), поэтому сразу не root и без capabilities.
|
||||
user: "1000:1000"
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
healthcheck:
|
||||
# Готовность к ЗАПИСИ: после рестарта мастер ~15 c отвечает "master is warming up" на
|
||||
# /dir/assign, хотя /cluster/healthz уже 200. Проверяем, что мастер выдаёт fid.
|
||||
test: ["CMD-SHELL", "wget -q -O - 'http://127.0.0.1:9333/dir/assign?count=1' | grep -q '\"fid\"'"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 15s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
hostname: rabbitmq
|
||||
user: rabbitmq
|
||||
environment:
|
||||
RABBITMQ_NODENAME: rabbit@localhost # distribution только на loopback (infra/rabbitmq.conf)
|
||||
ERL_EPMD_ADDRESS: 127.0.0.1
|
||||
volumes:
|
||||
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
|
||||
- ./infra/rabbitmq-enabled-plugins:/etc/rabbitmq/enabled_plugins:ro
|
||||
- ./secrets/pki/rabbitmq:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
secrets: [rabbitmq_definitions]
|
||||
networks: [data]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 24
|
||||
start_period: 20s
|
||||
start_interval: 2s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 512}
|
||||
|
||||
# ---------------- bootstrap ----------------
|
||||
migrate:
|
||||
build: *app-build
|
||||
command: ["python", "migrate.py"]
|
||||
environment:
|
||||
<<: [*tls-env, *db-env]
|
||||
BAO_ROLE: migrate
|
||||
DB_USER: messenger
|
||||
DB_PASSWORD_KEY: db_owner_password
|
||||
volumes:
|
||||
- ./secrets/pki/migrate:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [data, secrets]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||||
|
||||
s3-init:
|
||||
build: *app-build
|
||||
command: ["python", "s3_init.py"]
|
||||
environment:
|
||||
<<: *tls-env
|
||||
BAO_ROLE: s3-init
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
volumes:
|
||||
- ./secrets/pki/s3-init:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [data, secrets]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||||
|
||||
# ---------------- приложение ----------------
|
||||
api:
|
||||
build: *app-build
|
||||
env_file: .env
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *otel-env]
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8443"
|
||||
API_TLS: "1"
|
||||
API_ALLOWED_CLIENTS: lb # identity, которой разрешено подключаться к API (mTLS)
|
||||
BAO_ROLE: api
|
||||
DB_USER: messenger_api
|
||||
REDIS_HOST: redis
|
||||
REDIS_USER: api
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
BROKER_URL: "" # у API нет доступа к брокеру: задание уходит через outbox
|
||||
UPLOAD_TMP_DIR: /tmp/uploads
|
||||
# Буферы Argon2id (32 MiB) — всегда через mmap и сразу назад ОС. Без этого glibc оставляет освобождённые
|
||||
# буферы в per-thread arenas пула потоков Starlette: замер (login_dos.py, 200 проверок) — RSS +548 MiB
|
||||
# против +37 MiB с этой настройкой; при лимите контейнера 512 MiB это разница между OOM и запасом.
|
||||
MALLOC_MMAP_THRESHOLD_: "1048576"
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
volumes:
|
||||
- ./secrets/pki/api:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [frontend, data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
# Временные файлы текущих загрузок (до 10 MiB каждый) — tmpfs, учитывается в лимите памяти контейнера.
|
||||
tmpfs: ["/tmp:size=160m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
janitor:
|
||||
build: *app-build
|
||||
command: ["python", "janitor.py"]
|
||||
env_file: .env
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *otel-env]
|
||||
BAO_ROLE: janitor
|
||||
DB_USER: messenger_janitor
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
BROKER_URL: ""
|
||||
OTEL_SERVICE_NAME: messenger-janitor
|
||||
volumes:
|
||||
- ./secrets/pki/janitor:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 192M, pids: 256}
|
||||
|
||||
dispatcher:
|
||||
build: *app-build
|
||||
command: ["python", "dispatcher.py"]
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
|
||||
BAO_ROLE: dispatcher
|
||||
DB_USER: messenger_dispatcher
|
||||
BROKER_URL: amqps://rabbitmq:5671/messenger # не секрет: вход по сертификату CN=dispatcher
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
volumes:
|
||||
- ./secrets/pki/dispatcher:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- faults:/faults
|
||||
networks: [data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||||
|
||||
# Worker: обработка изображения в дочернем процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB,
|
||||
# RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS; контейнер ограничен по CPU/RAM/PID.
|
||||
worker:
|
||||
build: *app-build
|
||||
command: ["python", "worker.py"]
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
|
||||
BAO_ROLE: worker
|
||||
DB_USER: messenger_worker
|
||||
BROKER_URL: amqps://rabbitmq:5671/messenger # вход по сертификату CN=worker
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
|
||||
WORKER_TMP_DIR: /tmp/work
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
volumes:
|
||||
- ./secrets/pki/worker:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- faults:/faults
|
||||
networks: [data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=256m,mode=1777"]
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 70s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "2.00", memory: 1536M, pids: 128}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.25-alpine
|
||||
ports:
|
||||
- "127.0.0.1:8443:8443"
|
||||
- "127.0.0.1:8404:8404"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/pki/ingress:/run/tls-ingress:ro
|
||||
- ./secrets/pki/lb:/run/tls-lb:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [ingress, frontend]
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||||
|
||||
# ---------------- наблюдаемость ----------------
|
||||
# otel-gate владеет network namespace; Collector живёт в нём же и слушает только 127.0.0.1.
|
||||
# Имя otel-collector в сети telemetry указывает на gate (alias): приложение ходит на https://otel-collector:4318.
|
||||
otel-gate:
|
||||
image: haproxy:3.2.25-alpine
|
||||
volumes:
|
||||
- ./infra/otel/gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/pki/otel-collector:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks:
|
||||
telemetry:
|
||||
aliases: [otel-collector]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/gate-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 64M, pids: 256}
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.164.0
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
network_mode: "service:otel-gate"
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
- ./secrets/pki/otel-collector:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
depends_on:
|
||||
otel-gate: {condition: service_started}
|
||||
<<: *hardening
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 384M, pids: 256}
|
||||
|
||||
# Одноразовый chown volumes Loki/Tempo для uid 10001 (их процессы — не root). Исключение: root + CAP_CHOWN,
|
||||
# без сети, завершается сразу. Идемпотентно: при повторном up каталоги уже 10001 и внутри — 0700-каталоги
|
||||
# Loki/Tempo, в которые root без CAP_DAC_READ_SEARCH не войдёт, поэтому chown только для ещё чужого volume.
|
||||
telemetry-init:
|
||||
image: busybox:1.37.0
|
||||
user: root
|
||||
command: ["sh", "-c", "for d in /loki /tempo; do [ \"$$(stat -c %u $$d)\" = 10001 ] || chown -R 10001:10001 $$d || exit 1; done"]
|
||||
volumes:
|
||||
- lokidata:/loki
|
||||
- tempodata:/tempo
|
||||
network_mode: none
|
||||
cap_drop: [ALL]
|
||||
cap_add: [CHOWN]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 32M, pids: 256}
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.15.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --web.config.file=/etc/prometheus/web.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=1GB
|
||||
- --web.enable-otlp-receiver
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro
|
||||
- ./infra/prometheus/promtool-http.yml:/etc/prometheus/promtool-http.yml:ro
|
||||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- ./infra/prometheus/tests:/etc/prometheus/tests:ro
|
||||
- ./secrets/pki/prometheus:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- promdata:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090"
|
||||
networks: [telemetry, ops]
|
||||
<<: *hardening
|
||||
# promtool test rules (unit-тесты алертов, check_configs.py) создаёт временную TSDB в /tmp
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD", "promtool", "check", "ready", "--url=https://localhost:9090", "--http.config.file=/etc/prometheus/promtool-http.yml"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.9
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- ./secrets/pki/loki:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- lokidata:/loki
|
||||
ports:
|
||||
- "127.0.0.1:3100:3100"
|
||||
networks: [telemetry, ops]
|
||||
depends_on:
|
||||
telemetry-init: {condition: service_completed_successfully}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:3.1.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- ./secrets/pki/tempo:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- tempodata:/var/tempo
|
||||
ports:
|
||||
- "127.0.0.1:3200:3200"
|
||||
networks: [telemetry, ops]
|
||||
depends_on:
|
||||
telemetry-init: {condition: service_completed_successfully}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.4.14
|
||||
environment:
|
||||
GF_SERVER_PROTOCOL: https
|
||||
GF_SERVER_CERT_FILE: /run/tls/tls.crt
|
||||
GF_SERVER_CERT_KEY: /run/tls/tls.key
|
||||
GF_SERVER_DOMAIN: localhost
|
||||
GF_SERVER_ROOT_URL: https://localhost:3000/
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_SECURITY_COOKIE_SECURE: "true"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
|
||||
secrets: [grafana_admin_password]
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
|
||||
- ./secrets/pki/grafana:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
networks: [telemetry, ops]
|
||||
depends_on:
|
||||
prometheus: {condition: service_healthy}
|
||||
loki: {condition: service_started}
|
||||
tempo: {condition: service_started}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS -m 2 -o /dev/null --cacert /run/tls-ca/ca.crt https://localhost:3000/api/health || wget -q -T 2 -O /dev/null https://localhost:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 384M, pids: 256}
|
||||
|
||||
# ---------------- только для проверок (профиль tools, сам не стартует) ----------------
|
||||
# Контейнер-зонд во всех внутренних сетях: отрицательные/положительные проверки trust matrix изнутри
|
||||
# (student-tests/tls_probe.py). Получает ТОЛЬКО тестовые сертификаты и сертификаты клиентов, которыми
|
||||
# пробует войти «не туда»; ключа CA и секретов OpenBao у него нет.
|
||||
probe:
|
||||
build: *app-build
|
||||
profiles: [tools]
|
||||
command: ["python", "/probe/tls_probe.py", "matrix"]
|
||||
environment:
|
||||
PKI: /pki
|
||||
PYTHONPATH: /app
|
||||
volumes:
|
||||
- ./student-tests:/probe:ro
|
||||
- ./secrets/pki/public:/pki/public:ro
|
||||
- ./secrets/pki/test:/pki/test:ro
|
||||
- ./secrets/pki/lb:/pki/lb:ro
|
||||
- ./secrets/pki/api:/pki/api:ro
|
||||
- ./secrets/pki/worker:/pki/worker:ro
|
||||
- ./secrets/pki/dispatcher:/pki/dispatcher:ro
|
||||
- ./secrets/pki/janitor:/pki/janitor:ro
|
||||
- ./secrets/pki/grafana:/pki/grafana:ro
|
||||
- ./secrets/pki/otel-collector:/pki/otel-collector:ro
|
||||
- ./secrets/pki/operator:/pki/operator:ro
|
||||
networks: [frontend, data, secrets, telemetry]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||||
|
||||
networks:
|
||||
ingress: {}
|
||||
frontend:
|
||||
internal: true
|
||||
data:
|
||||
internal: true
|
||||
secrets:
|
||||
internal: true
|
||||
telemetry:
|
||||
internal: true
|
||||
# Только для публикации админ-интерфейсов на 127.0.0.1 (internal-сеть не может публиковать порты).
|
||||
ops: {}
|
||||
|
||||
volumes:
|
||||
openbaodata: {}
|
||||
pgdata: {}
|
||||
redisdata: {}
|
||||
s3data: {}
|
||||
rabbitmqdata: {}
|
||||
faults: {}
|
||||
promdata: {}
|
||||
lokidata: {}
|
||||
tempodata: {}
|
||||
grafanadata: {}
|
||||
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
s3_config: {file: ./secrets/s3_config.json}
|
||||
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
|
||||
grafana_admin_password: {file: ./secrets/grafana_admin_password}
|
||||
+38
-20
@@ -1,32 +1,43 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия — решение lab7 (Бабиченко, БВТ2303)
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
|
||||
**Общее для всех строк.** Один учебный корневой CA оператора (`scripts/pki.py`, ECDSA P-256, 365 суток; ключ CA — только `secrets/pki/ca/ca.key` у оператора, ни в один контейнер не монтируется). Leaf-сертификаты сервисов — 30 суток, identity = Subject CN (+ URI SAN `spiffe://messenger.lab/<cn>`), DNS SAN = имя сервиса в compose. Отзыв — CRL `secrets/pki/public/crl.pem` (подписан CA, nextUpdate 30 суток). Каждый контейнер монтирует только **свой** каталог `secrets/pki/<identity>` и публичный `secrets/pki/public` (read-only). Клиенты проверяют цепочку до CA и **имя сервера**; в коде нет ветки «без проверки». Отрицательные проверки каждой строки выполняет `python student-tests/test_tls_matrix.py` (контейнер `probe` во всех внутренних сетях, только тестовые и «чужие» клиентские сертификаты, без ключа CA и без секретов) — результат `evidence/tls-matrix.json`; права identity — `python student-tests/test_least_privilege.py` (`evidence/least-privilege.json`).
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Клиент → ingress/LB | **127.0.0.1:8443**, HTTPS (HAProxy 3.2, сертификат `ingress`: SAN localhost/127.0.0.1, TLS 1.2+, без session tickets), HSTS; бизнес-доступ — Bearer (сессия 256 бит, в Redis только SHA-256). HTTP :8080 больше не слушается | Expiry/revocation сессии: абсолютный TTL в Redis (`SESSION_TTL_SECONDS`), logout = DEL ключа, одинаково на всех репликах. Общий для реплик rate limit входа/регистрации в Redis (10 попыток/мин на username, 600/мин на IP) → 429 + Retry-After до Argon2id; бюджет хешей реплики → 503 + Retry-After | `infra/haproxy.cfg` (frontend api_in); `make test-security CA_FILE=...` (пустой trust store → ошибка проверки), `test_lab7_security.py`: чужой CA и неверное имя → `SSLCertVerificationError`, `test_login_rate_limit_is_shared`, `test_logout_revokes_token`; `test_expiry.py` (lab2) |
|
||||
| LB → API (все реплики) | **api:8443**, mTLS: сервер — `api` (SAN api, HAProxy `verify required ca-file crl-file verifyhost api`), клиент — leaf `lb`. API (uvicorn, `tlsutil.py`): CERT_REQUIRED + CA (цепочка, срок), затем **allowlist CN=lb** (`API_ALLOWED_CLIENTS`) и CRL до чтения запроса, иначе abort. Порт API не опубликован, сеть `frontend` internal; health-listener 127.0.0.1:8081 только внутри контейнера, только /health/* | Ротация leaf `lb`: `pki.py rotate lb` → `docker compose kill -s USR2 lb` (reload без обрыва) → `pki.py revoke lb --previous`: API перечитывает CRL каждые 5 с (подпись CRL проверяется ключом CA) и **закрывает уже открытые соединения** с отозванным сертификатом; session tickets выключены | `tls_probe.py lb-api`: нет сертификата / чужой CA с CN=lb / истёкший lb / intruder (наш CA) / worker / api → отказ; lb → 200; клиент, доверяющий только чужому CA → не принимает сервер; plaintext → нет ответа. `lifecycle.py rotate-leaf` (старый leaf → отказ, новый → 200, нагрузка без ошибок) |
|
||||
| API/worker → PostgreSQL | **postgres:5432**, TLS (`ssl=on`, TLS 1.2+), клиенты `sslmode=verify-full` + CA (имя postgres); `pg_hba`: только `hostssl … scram-sha-256`, `hostnossl … reject`. Роли: `messenger_api`, `messenger_worker`, `messenger_dispatcher`, `messenger_janitor` (GRANT по запросам, `migrations/006_roles.sql`), владелец `messenger` — только migrate | Пароли ролей — из OpenBao (`messenger/<сервис>.db_password`), берутся на каждое новое соединение пула; migrate выставляет их как SCRAM-verifier (открытый пароль не попадает ни в SQL, ни в журнал сервера). Ротация пароля роли (процедура, не демонстрировалась — на 4 показана ротация пароля Redis): новая версия в KV → `docker compose run --rm migrate` выставляет её роли → сервисы перечитывают KV, новые соединения — с новым паролем; окно до 30 с, когда новые соединения со старым паролем получают отказ (пул держит открытые). mTLS для PostgreSQL не включён (допустимое усиление, не требование) | `infra/postgres/pg_hba.conf`, `dbconn.py`; `tls_probe.py postgres`: verify-full → дошли до SCRAM (TLS ок), `sslmode=disable` → `no pg_hba.conf entry … no encryption`, доверие чужому CA / подключение по IP → отказ проверки сервера. `test_least_privilege.py`: worker не читает users/messages, api не DELETE/DROP/CREATE/ALTER ROLE и т.д. |
|
||||
| API/worker → S3 | **s3:8333** HTTPS (SeaweedFS 4.52 от uid 1000, сертификат s3; одновременных тел загрузок ≤ 128 MiB, остальные в очереди); master/volume/filer слушают только 127.0.0.1 контейнера; Iceberg/Lance/IAM API выключены; gRPC шлюза :18333 — mTLS, только CN=s3 (`infra/seaweedfs/security.toml`). Ключи: s3-init (admin), api, worker, janitor — разные, из OpenBao | Права по prefix (`s3_config`): api — PUT/DELETE `staging/`, GET `attachments/`,`originals/`; worker — GET `staging/`,`originals/`, PUT/GET/LIST `attachments/`, DELETE исходника; janitor — LIST и DELETE, без GET; никто кроме s3-init не создаёт bucket | `tls_probe.py s3`: аноним по HTTPS → 403, чужой CA → отказ проверки, plaintext → 400 «HTTP to HTTPS», gRPC :18333 с CN=intruder/api/без сертификата → bad certificate / certificate required. `test_least_privilege.py`: api → PUT attachments/, GET staging/, LIST → AccessDenied; worker → LIST корня, PUT вне prefix → AccessDenied; janitor → GET → AccessDenied |
|
||||
| API/worker → broker | **rabbitmq:5671** AMQPS (порт 5672 выключен), `verify_peer` + `fail_if_no_peer_cert`; вход механизмом **EXTERNAL**: пользователь = CN сертификата (`worker`, `dispatcher`), паролей нет; права: worker — read основной очереди, dispatcher — write в exchange + read DLQ, configure ни у кого. У API брокера нет (outbox) | mTLS с ограничением identity: сертификат нашего CA с CN без пользователя брокера (api, lb, intruder) → ACCESS_REFUSED при входе; worker не публикует, dispatcher не читает задания, никто не объявляет очереди. Erlang distribution и epmd — только 127.0.0.1 | `infra/rabbitmq.conf`, `rabbitmq-enabled-plugins`; `tls_probe.py broker`: worker/dispatcher → вход, api/intruder/lb → login refused, чужой CA / без сертификата → отказ TLS, worker publish / dispatcher basic_get / queue_declare → 403 |
|
||||
| API/worker → Collector | **otel-collector:4318** (alias otel-gate) OTLP/HTTP по mTLS: шлюз HAProxy проверяет CA, срок, CRL и **CN ∈ {api, worker, dispatcher, janitor}**, иначе 403; Collector слушает только 127.0.0.1:4319 в общем network namespace со шлюзом (локальный hop не виден из сети) | Раздельные identity: каждый сервис шлёт со своим сертификатом (`OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`); сертификат того же CA, но не приложения (lb, grafana, operator, intruder) → 403. Метрики самого Collector — :8889 через шлюз только для CN prometheus/operator | `infra/otel/gate.cfg`, `infra/otel/collector.yaml`; `tls_probe.py collector`: 4 app-identity → 200, lb/grafana/operator/intruder → 403, чужой CA / без сертификата → отказ TLS, :8889 с CN=api → 403 |
|
||||
| Collector → telemetry backends | Collector → Tempo :4317 (OTLP gRPC, TLS, `client_ca_file`), → Loki :3100 и Prometheus :9090 (HTTPS); везде клиентский сертификат CN=otel-collector и проверка сервера по CA и имени. Prometheus scrape — тоже HTTPS + сертификат prometheus | Prometheus: `client_allowed_sans` — пишут/читают только otel-collector, grafana, prometheus, operator (api, intruder того же CA → отказ на рукопожатии). Loki/Tempo различают только «наш CA / не наш» — разделить чтение и запись по identity они не умеют (остаточный риск, сеть telemetry internal). Ротация leaf otel-collector: Prometheus перечитывает web-config на каждый запрос; Collector/Loki/Tempo — рестарт сервиса | `infra/prometheus/web.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` (в т.ч. внутренний gRPC Tempo по TLS), `tls_probe.py telemetry` |
|
||||
| Grafana → data sources | HTTPS к Prometheus/Loki/Tempo с проверкой CA (`tlsAuthWithCACert`) и клиентским сертификатом CN=grafana (`$__file{}` в provisioning, в Git сертификатов нет) | Отрицательные проверки: без сертификата / чужой CA к Loki/Tempo/Prometheus → отказ; к Prometheus — только разрешённые SAN | `infra/grafana/provisioning/datasources/datasources.yaml`; `check_configs.py` (health источников = OK), `tls_probe.py telemetry` |
|
||||
| API/worker/agent → OpenBao | **openbao:8200** TLS (сертификат openbao), вход **cert auth**: роль = CN, `allowed_common_names=[роль]`, доверие только нашему CA, CRL загружен в cert auth. Токен: service, TTL 15 мин, renewable, max 1 ч; policy — read только `messenger/data/<роль>` | Renewal: `secretstore.py` продлевает токен (renew-self с тем же сертификатом), при отказе/исчерпании max TTL — новый вход сертификатом; KV перечитывается раз в 30 с (подхват ротации). Отзыв: `bao.py revoke-service-tokens` + CRL в cert auth (отозванный leaf не войдёт). Sealed/недоступен: работающий процесс живёт на секретах в памяти, новый — не стартует (fail closed) | `infra/openbao/config.hcl`, `infra/openbao/policies/*.hcl`, `scripts/bao.py`; `tls_probe.py openbao`: worker как api / intruder / чужой CA / без сертификата → 400; `test_least_privilege.py`: чужие пути, запись KV/политики, список auth, создание токена → 403; `lifecycle.py token-expiry`, `lifecycle.py sealed` |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | OpenBao: 127.0.0.1:8200 HTTPS + root token оператора (файл `secrets/operator/openbao-init.json`, 0600, вне Git и контейнеров). Grafana: https://localhost:3000, admin + пароль. Prometheus/Loki/Tempo: 127.0.0.1, mTLS сертификатом `operator`. HAProxy stats: 127.0.0.1:8404, mTLS, только CN=operator. Admin-ключ S3 — только у s3-init через OpenBao, консоли S3 нет | Роли разделены: приложение не имеет operator-прав нигде (нет root token, admin S3, владельца БД, admin Redis); оператор не использует ключи сервисов. Redis admin — только у оператора для ротации (`lifecycle.py`) | `check_stack.py`/`check_hardening.py`: опубликованы только 8443, 8404, 8200, 9090, 3100, 3200, 3000 и только на 127.0.0.1; `tls_probe.py telemetry` (без сертификата оператора → отказ) |
|
||||
| API → Redis/Valkey, если используется | **redis:6379** только TLS (`port 0`), **tls-auth-clients yes** (клиентский сертификат нашего CA обязателен), затем ACL: `default off`, `api` — только ключи `session:*`, `ratelimit:*` и команды get/set/del/incr/incrby/pexpire/pttl/ping/multi/exec, без pub/sub; `admin` — оператор | Ротация пароля `api` без рестартов: временно два пароля (`ACL SETUSER api >new`) → новая версия в OpenBao → все реплики прочитали → `ACL SETUSER api resetpass >new` + `CLIENT KILL USER api` + `ACL SAVE`. mTLS включён (identity на уровне TLS — любой сертификат нашего CA, права — ACL) | `infra/redis.conf`, `scripts/init_secrets.py` (ACL), `auth.py`; `tls_probe.py redis`: наш CA без AUTH → NOAUTH, неверный пароль → WRONGPASS, без сертификата / чужой CA → отказ TLS, plaintext → сброс; `test_least_privilege.py`: KEYS, FLUSHALL, CONFIG, ACL, PUBLISH, чужие ключи → NOPERM; `lifecycle.py rotate-secret` |
|
||||
| Replication/кластерные каналы, если добавлены | N/A: репликации и кластеров нет (по одному PostgreSQL, Redis, RabbitMQ, SeaweedFS). Внутрипроцессные каналы закрыты: SeaweedFS master/volume/filer — 127.0.0.1 + gRPC mTLS (CN=s3); RabbitMQ distribution/epmd — 127.0.0.1; Loki gRPC — 127.0.0.1; Tempo внутренний gRPC — TLS с клиентским сертификатом | Модель отказа: один Docker-host, отказ узла не выдерживается (см. REPORT.md, ограничения) | `tls_probe.py s3` (gRPC 18333), `infra/seaweedfs/security.toml`, `infra/rabbitmq.conf` |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
## Служебные каналы (scrape / health / bootstrap)
|
||||
|
||||
| Канал | Как защищён | Почему не обход |
|
||||
| --- | --- | --- |
|
||||
| Healthcheck API | `http://127.0.0.1:8081/health/{live,ready}` — отдельный listener **внутри контейнера** | loopback контейнера недоступен из сетей; бизнес-путей нет (404) |
|
||||
| Healthcheck LB / otel-gate | `127.0.0.1:8405` внутри контейнера (monitor-uri) | то же |
|
||||
| Healthcheck PostgreSQL / Redis / RabbitMQ / SeaweedFS | unix-socket `pg_isready`; `redis-cli --tls` без AUTH (ответ NOAUTH); `rabbitmq-diagnostics` по loopback-distribution; `wget http://127.0.0.1:9333/dir/assign` (master на loopback; готовность к записи, а не только чтению — после рестарта мастер ~15 с не выдаёт fid) | всё внутри контейнера |
|
||||
| Healthcheck Prometheus / Grafana / OpenBao | `promtool check ready --http.config.file` (mTLS), `curl --cacert https://localhost:3000`, `bao status -ca-cert` | TLS и внутри контейнера |
|
||||
| Prometheus scrape | HTTPS + сертификат prometheus к Collector (через gate, CN-allowlist), Loki, Tempo, себе | см. строку Collector → backends |
|
||||
| Bootstrap | migrate (CN=migrate: пароль владельца БД + пароли ролей), s3-init (CN=s3-init: admin S3) — одноразовые, свои политики OpenBao; оператор — `pki.py`, `init_secrets.py`, `bao.py` на хосте | значения не печатаются; файл с секретами приложений удаляется после загрузки в OpenBao |
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». → `make test-security`, `test_lab7_security.py test_https_only_and_certificate_checks`.
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. → `test_tls_matrix.py lb-api` (+ истёкший lb, сертификат самого api, отозванный предыдущий leaf).
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. → `test_tls_matrix.py broker collector` (identity проверяется самим брокером — EXTERNAL/permissions — и шлюзом Collector по CN).
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. → `test_lab7_security.py test_outsider_cannot_reach_foreign_chat`, `test_api.py` (lab2).
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. → `test_least_privilege.py` (openbao: 5 чужих путей, запись KV, политика, список auth, токен; «own secret keys (no admin credentials)»).
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. → `lifecycle.py rotate-leaf` (новое подключение старым leaf — отказ; API закрывает открытые соединения со старым leaf; tickets выключены), `lifecycle.py rotate-secret` (`CLIENT KILL USER api` + проверка, что активен только новый пароль), `lifecycle.py token-expiry`.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
|
||||
@@ -35,3 +46,10 @@
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
|
||||
**Решение.**
|
||||
- *Кто владеет чем.* Оператор (студент на своём хосте) — ключ CA (`secrets/pki/ca/`), доли unseal и root token (`secrets/operator/openbao-init.json`), пароль admin Redis и владельца БД (`secrets/operator/`). Каталог `secrets/` 0700, файлы оператора 0600, ни один не смонтирован в контейнер. В реальной эксплуатации: `bao.py init --print-shares-once` раздаёт 3 доли разным людям (порог 2) и ничего не сохраняет, `bao.py unseal --prompt`; root token отзывается после настройки; CA — офлайн.
|
||||
- *Первый секрет сервиса* — его leaf-сертификат (30 суток, отзывается CRL, меняется без пересборки образа). Постоянного «секрета для получения секретов» (SecretID/BAO_TOKEN в env) нет: процесс входит в OpenBao методом cert и получает токен на 15 мин.
|
||||
- *Рестарт API* — без участия оператора (сертификат на месте, OpenBao распечатан). *Рестарт OpenBao* — он sealed: `python scripts/bao.py unseal` (restart-hook делает это сам, читая доли из файла оператора; ничего не печатается). Пока sealed: работающие сервисы обслуживают запросы с секретами из памяти, новые/перезапущенные не становятся ready.
|
||||
- *Renewal.* Leaf — `pki.py rotate <identity>` + reload/restart сервиса (каталоги смонтированы, образы не меняются), старый — `pki.py revoke <identity> --previous` + `bao.py sync-crl` + reload lb/otel-gate; CRL переподписывается `pki.py crl` раньше nextUpdate (30 суток). Токены — автоматически (renew/relogin). Рабочие секреты — новая версия KV, сервисы перечитывают её сами.
|
||||
- *Восстановление.* Потеря OpenBao volume = потеря секретов приложений → `init_secrets.py` после удаления `secrets/s3_config.json`/`redis/users.acl` и пересоздание volumes (учебные данные). Потеря CA → новый CA и все leaf (`pki.py init` в чистом `secrets/pki`), `bao.py configure` (роли cert auth получают новый CA).
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
import logging
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
import dbconn
|
||||
import pipeline
|
||||
import settings
|
||||
import telemetry
|
||||
from errors import bad_request, conflict, not_found, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
|
||||
def check_connection(conn) -> None:
|
||||
"""Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса."""
|
||||
from opentelemetry.instrumentation.utils import suppress_instrumentation
|
||||
with suppress_instrumentation():
|
||||
ConnectionPool.check_connection(conn)
|
||||
|
||||
|
||||
def pool_kwargs() -> dict:
|
||||
"""lab7: вызывается на КАЖДОЕ новое соединение пула — пароль роли берётся из OpenBao (текущая версия)."""
|
||||
return dbconn.connect_kwargs(
|
||||
row_factory=dict_row,
|
||||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
|
||||
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
|
||||
)
|
||||
|
||||
|
||||
pool = ConnectionPool(
|
||||
conninfo="",
|
||||
min_size=settings.DB_POOL_MIN,
|
||||
max_size=settings.DB_POOL_MAX,
|
||||
timeout=settings.DB_POOL_WAIT_SECONDS,
|
||||
kwargs=pool_kwargs,
|
||||
check=check_connection,
|
||||
open=False,
|
||||
)
|
||||
|
||||
|
||||
# lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»).
|
||||
POOL_NAME = {"db.client.connection.pool.name": "messenger-api"}
|
||||
POOL_WAIT = telemetry.meter.create_histogram(
|
||||
"db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool")
|
||||
|
||||
|
||||
def _pool_stats():
|
||||
try:
|
||||
return pool.get_stats()
|
||||
except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет
|
||||
return None
|
||||
|
||||
|
||||
def _observe_connections(_options):
|
||||
from opentelemetry.metrics import Observation
|
||||
stats = _pool_stats()
|
||||
if not stats:
|
||||
return []
|
||||
size, available = stats.get("pool_size", 0), stats.get("pool_available", 0)
|
||||
return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}),
|
||||
Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})]
|
||||
|
||||
|
||||
def _observe_stat(key):
|
||||
def callback(_options):
|
||||
from opentelemetry.metrics import Observation
|
||||
stats = _pool_stats()
|
||||
return [Observation(stats.get(key, 0), POOL_NAME)] if stats else []
|
||||
return callback
|
||||
|
||||
|
||||
telemetry.meter.create_observable_up_down_counter(
|
||||
"db.client.connection.count", [_observe_connections], unit="{connection}",
|
||||
description="Connections in the pool by state (used/idle)")
|
||||
telemetry.meter.create_observable_up_down_counter(
|
||||
"db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}",
|
||||
description="Maximum pool size (DB_POOL_MAX)")
|
||||
telemetry.meter.create_observable_up_down_counter(
|
||||
"db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}",
|
||||
description="Requests waiting for a free connection now")
|
||||
telemetry.meter.create_observable_counter(
|
||||
"db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}",
|
||||
description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection(wait_seconds: float | None = None):
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with pool.connection(timeout=wait_seconds) as conn:
|
||||
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
|
||||
yield conn
|
||||
except PoolTimeout:
|
||||
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
|
||||
raise
|
||||
except (psycopg.OperationalError, PoolTimeout) as exc:
|
||||
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__,
|
||||
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
|
||||
raise unavailable() from None
|
||||
except psycopg.DataError as exc:
|
||||
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__,
|
||||
extra={"event": "db.rejected_value", "error.type": type(exc).__name__})
|
||||
raise bad_request("Invalid request data") from None
|
||||
|
||||
|
||||
TEXT_FIELDS = ("display_name", "title", "text", "filename")
|
||||
|
||||
|
||||
def to_db(value: str) -> bytes:
|
||||
return value.encode("utf-8")
|
||||
|
||||
|
||||
def from_db(row: dict | None) -> dict | None:
|
||||
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
|
||||
if row is None:
|
||||
return None
|
||||
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
|
||||
|
||||
|
||||
def ping() -> None:
|
||||
with connection(wait_seconds=1) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
|
||||
MIGRATIONS_LOCK = 20260202
|
||||
|
||||
|
||||
def migration_files() -> list:
|
||||
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
|
||||
|
||||
|
||||
def run_migrations() -> list[str]:
|
||||
files = migration_files()
|
||||
applied = []
|
||||
with dbconn.connect(autocommit=True) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
|
||||
try:
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now())""")
|
||||
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for path in files:
|
||||
if path.name in done:
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
|
||||
applied.append(path.name)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
|
||||
return applied
|
||||
|
||||
|
||||
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
try:
|
||||
applied = run_migrations()
|
||||
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
|
||||
return
|
||||
except psycopg.OperationalError:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log.warning("PostgreSQL is not ready yet, retry in 1 s")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def missing_migrations() -> list[str]:
|
||||
expected = [path.name for path in migration_files()]
|
||||
with dbconn.connect(autocommit=True) as conn:
|
||||
try:
|
||||
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
except (pg_errors.UndefinedTable, pg_errors.InsufficientPrivilege):
|
||||
# lab7: права ролей выдаёт миграция 006 — до неё схема для приложения «не готова».
|
||||
return expected
|
||||
return [name for name in expected if name not in done]
|
||||
|
||||
|
||||
def wait_for_schema(max_wait_seconds: int) -> None:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
try:
|
||||
missing = missing_migrations()
|
||||
except psycopg.OperationalError:
|
||||
missing = None
|
||||
if missing == []:
|
||||
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
|
||||
return
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
|
||||
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
|
||||
else "missing " + ", ".join(missing))
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
|
||||
USER_COLUMNS = "id, username, display_name, created_at"
|
||||
|
||||
|
||||
def create_user(username: str, display_name: str, password_hash: str) -> dict:
|
||||
try:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(
|
||||
f"""INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
|
||||
(uuid4(), username, to_db(display_name), password_hash),
|
||||
).fetchone())
|
||||
except pg_errors.UniqueViolation:
|
||||
raise conflict("Username already exists") from None
|
||||
|
||||
|
||||
def get_user(user_id: UUID) -> dict | None:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
|
||||
|
||||
|
||||
def get_user_with_password(username: str) -> dict | None:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(
|
||||
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
|
||||
).fetchone())
|
||||
|
||||
|
||||
def update_password_hash(user_id: UUID, old_hash: str, new_hash: str) -> bool:
|
||||
"""lab7: rehash при входе (параметры Argon2id выросли). Условие на старый hash — без гонки с другим входом."""
|
||||
with connection() as conn:
|
||||
cursor = conn.execute("UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s",
|
||||
(new_hash, user_id, old_hash))
|
||||
return cursor.rowcount == 1
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
|
||||
|
||||
|
||||
def create_chat(title: str, member_ids: list[UUID]) -> dict:
|
||||
chat_id = uuid4()
|
||||
members = sorted(member_ids)
|
||||
try:
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||
(chat_id, to_db(title)),
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
[(chat_id, member_id) for member_id in members],
|
||||
)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise bad_request("Unknown member") from None
|
||||
return {**from_db(row), "member_ids": members}
|
||||
|
||||
|
||||
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
|
||||
with connection() as conn:
|
||||
chat = conn.execute(
|
||||
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
WHERE c.id = %s""",
|
||||
(user_id, chat_id),
|
||||
).fetchone()
|
||||
if chat is None:
|
||||
raise not_found("Chat not found")
|
||||
return from_db(chat)
|
||||
|
||||
|
||||
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
|
||||
where, params = "", [user_id]
|
||||
if after is not None:
|
||||
where = "WHERE (c.created_at, c.id) > (%s, %s)"
|
||||
params += list(after)
|
||||
with connection() as conn:
|
||||
rows = conn.execute(
|
||||
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
{where}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s""",
|
||||
(*params, limit + 1),
|
||||
).fetchall()
|
||||
return [from_db(row) for row in rows]
|
||||
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
|
||||
|
||||
|
||||
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
def check_member(chat_id: UUID, user_id: UUID) -> None:
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise not_found("Chat not found")
|
||||
|
||||
|
||||
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
|
||||
idempotency_key: str, request_hash: bytes) -> dict:
|
||||
try:
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
|
||||
raise not_found("Chat not found")
|
||||
if attachment_ids:
|
||||
# Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409).
|
||||
# FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние.
|
||||
found = conn.execute(
|
||||
"""SELECT id, state FROM attachments
|
||||
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""",
|
||||
(attachment_ids, chat_id),
|
||||
).fetchall()
|
||||
if len(found) != len(attachment_ids):
|
||||
raise not_found("Attachment not found")
|
||||
failed = any(row["state"] == "failed" for row in found)
|
||||
else:
|
||||
failed = False
|
||||
saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
|
||||
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s"""
|
||||
if failed:
|
||||
# Повтор уже созданного сообщения возвращает прежний 201, даже если вложение
|
||||
# с тех пор стало failed. Новое сообщение с failed-вложением — 409.
|
||||
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
|
||||
if saved is None:
|
||||
raise conflict("Failed attachment cannot be attached")
|
||||
created = None
|
||||
else:
|
||||
created = conn.execute(
|
||||
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
|
||||
RETURNING id, chat_id, sender_id, text, created_at""",
|
||||
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
|
||||
).fetchone()
|
||||
if created is None:
|
||||
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
|
||||
if created is not None:
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
VALUES (%s, %s, %s, %s)""",
|
||||
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
|
||||
)
|
||||
return {**from_db(created), "attachment_ids": list(attachment_ids)}
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise not_found("Chat not found") from None
|
||||
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
|
||||
raise unavailable()
|
||||
if bytes(saved.pop("request_hash")) != request_hash:
|
||||
raise conflict("Idempotency-Key was already used with a different request")
|
||||
return from_db(saved)
|
||||
|
||||
|
||||
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
|
||||
where, params = "m.chat_id = %s", [chat_id]
|
||||
if after is not None:
|
||||
where += " AND (m.created_at, m.id) > (%s, %s)"
|
||||
params += list(after)
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise not_found("Chat not found")
|
||||
rows = conn.execute(
|
||||
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
|
||||
WHERE {where}
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT %s""",
|
||||
(*params, limit + 1),
|
||||
).fetchall()
|
||||
return [from_db(row) for row in rows]
|
||||
|
||||
|
||||
# ---------- вложения ----------
|
||||
|
||||
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, "
|
||||
"pipeline_version, created_at, updated_at, object_key")
|
||||
VISIBLE_STATES = ("queued", "processing", "ready", "failed")
|
||||
|
||||
|
||||
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
|
||||
size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None:
|
||||
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging.
|
||||
|
||||
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
|
||||
и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД).
|
||||
"""
|
||||
try:
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
width, height, source_key, state)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
|
||||
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
|
||||
width, height, source_key),
|
||||
)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise not_found("Chat not found") from None
|
||||
|
||||
|
||||
def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None:
|
||||
"""Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией.
|
||||
|
||||
После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL,
|
||||
сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace).
|
||||
"""
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
f"""UPDATE attachments SET state = 'queued', updated_at = now()
|
||||
WHERE id = %s AND state = 'uploading'
|
||||
RETURNING {ATTACHMENT_COLUMNS}""",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
pipeline.enqueue_new(conn, attachment_id, actor)
|
||||
return from_db(row)
|
||||
|
||||
|
||||
def delete_upload_intent(attachment_id: UUID) -> None:
|
||||
with connection(wait_seconds=1) as conn:
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
|
||||
"""Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404.
|
||||
|
||||
Возвращает строку и варианты текущей версии pipeline (они есть только у ready).
|
||||
"""
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
|
||||
FROM attachments a
|
||||
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
|
||||
WHERE a.id = %s AND a.state = ANY(%s)""",
|
||||
(user_id, attachment_id, list(VISIBLE_STATES)),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found("Attachment not found")
|
||||
variants = []
|
||||
if row["state"] == "ready":
|
||||
variants = conn.execute(
|
||||
"""SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s
|
||||
ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""",
|
||||
(attachment_id, row["pipeline_version"]),
|
||||
).fetchall()
|
||||
return {**from_db(row), "variants": variants}
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Параметры подключения к PostgreSQL (lab7) — общие для api, worker, dispatcher, janitor, ops, migrate.
|
||||
|
||||
TLS verify-full: цепочка до учебного CA и имя сервера = DB_HOST (SAN сертификата postgres). На сервере
|
||||
pg_hba разрешает только hostssl + scram-sha-256, незашифрованное подключение отвергается.
|
||||
Роль у каждого сервиса своя (DB_USER: messenger_api, messenger_worker, ...), права — migrations/006_roles.sql.
|
||||
Пароль берётся из OpenBao при КАЖДОМ новом подключении: после ротации новые соединения пула используют
|
||||
новую версию, уже открытые продолжают работать до закрытия.
|
||||
"""
|
||||
import psycopg
|
||||
|
||||
import settings
|
||||
from secretstore import store
|
||||
|
||||
|
||||
def connect_kwargs(**extra) -> dict:
|
||||
params = {
|
||||
"host": settings.DB_HOST,
|
||||
"port": settings.DB_PORT,
|
||||
"dbname": settings.DB_NAME,
|
||||
"user": settings.DB_USER,
|
||||
"password": store.get(settings.DB_PASSWORD_KEY),
|
||||
"sslmode": settings.DB_SSLMODE,
|
||||
"sslrootcert": settings.TLS_CA_FILE,
|
||||
"sslcertmode": "disable", # клиентский сертификат PostgreSQL не нужен (аутентификация — SCRAM)
|
||||
"application_name": f"{settings.BAO_ROLE}@{settings.INSTANCE_ID}",
|
||||
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
}
|
||||
params.update(extra)
|
||||
return params
|
||||
|
||||
|
||||
def connect(**kwargs) -> psycopg.Connection:
|
||||
return psycopg.connect(**connect_kwargs(), **kwargs)
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
|
||||
|
||||
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
|
||||
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
|
||||
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
|
||||
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
|
||||
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
|
||||
|
||||
Цикл:
|
||||
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
|
||||
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
|
||||
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
|
||||
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
|
||||
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
|
||||
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
|
||||
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
|
||||
"""
|
||||
import logging
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from opentelemetry.trace import SpanKind
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import dbconn
|
||||
import secretstore
|
||||
import settings
|
||||
import telemetry
|
||||
|
||||
telemetry.setup("messenger-dispatcher")
|
||||
|
||||
import broker # noqa: E402
|
||||
import faults # noqa: E402
|
||||
import pipeline # noqa: E402
|
||||
|
||||
log = logging.getLogger("messenger.dispatcher")
|
||||
|
||||
# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs,
|
||||
# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому
|
||||
# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик
|
||||
# только читает последний снимок (не ходит в БД из потока экспорта).
|
||||
STATS_SECONDS = 5
|
||||
STATS = {"at": 0.0, "db": None, "broker": None}
|
||||
PUBLISHED = telemetry.meter.create_counter(
|
||||
"messenger.messages.published", unit="{message}",
|
||||
description="Outbox rows published to the broker (outcome: ok/error)")
|
||||
QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE}
|
||||
|
||||
|
||||
def _fresh(kind):
|
||||
return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None
|
||||
|
||||
|
||||
def _observe(kind, build):
|
||||
def callback(_options):
|
||||
from opentelemetry.metrics import Observation
|
||||
data = _fresh(kind)
|
||||
# lab7: ресурс, невидимый identity (RabbitMQ 4.2+), даёт None — такую точку не пишем
|
||||
return [Observation(value, attrs) for value, attrs in build(data) if value is not None] if data else []
|
||||
return callback
|
||||
|
||||
|
||||
telemetry.meter.create_observable_gauge(
|
||||
"messenger.queue.depth", [_observe("broker", lambda d: [
|
||||
(d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])],
|
||||
unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)")
|
||||
telemetry.meter.create_observable_gauge(
|
||||
"messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])],
|
||||
unit="{consumer}", description="Consumers subscribed to the queue")
|
||||
telemetry.meter.create_observable_gauge(
|
||||
"messenger.jobs.backlog", [_observe("db", lambda d: [
|
||||
(d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}),
|
||||
(d["processing"], {"status": "processing"})])],
|
||||
unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing")
|
||||
telemetry.meter.create_observable_gauge(
|
||||
"messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])],
|
||||
unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits")
|
||||
telemetry.meter.create_observable_gauge(
|
||||
"messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])],
|
||||
unit="{message}", description="Outbox rows due but not yet published")
|
||||
telemetry.meter.create_observable_gauge(
|
||||
"messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])],
|
||||
unit="s", description="Age of the oldest due unpublished outbox row (publish lag)")
|
||||
|
||||
|
||||
def collect_stats(conn, publisher) -> None:
|
||||
"""Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот."""
|
||||
if time.monotonic() - STATS["at"] < STATS_SECONDS:
|
||||
return
|
||||
db_stats = conn.execute(
|
||||
"""SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due,
|
||||
count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed,
|
||||
count(*) FILTER (WHERE status = 'processing') AS processing,
|
||||
coalesce(extract(epoch FROM now() - min(available_at)
|
||||
FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age
|
||||
FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone()
|
||||
outbox = conn.execute(
|
||||
"""SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due,
|
||||
coalesce(extract(epoch FROM now() - min(available_at)
|
||||
FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age
|
||||
FROM outbox WHERE published_at IS NULL""").fetchone()
|
||||
STATS["db"] = {**db_stats, **outbox}
|
||||
try:
|
||||
STATS["broker"] = broker.check_topology(publisher.open())
|
||||
except broker.BrokerError:
|
||||
STATS["broker"] = None
|
||||
publisher.close()
|
||||
STATS["at"] = time.monotonic()
|
||||
|
||||
STOP = threading.Event()
|
||||
ACTOR = "dispatcher@" + settings.INSTANCE_ID
|
||||
|
||||
|
||||
def connect_db():
|
||||
# lab7: TLS verify-full, своя роль PostgreSQL, пароль из OpenBao (dbconn)
|
||||
return dbconn.connect(autocommit=True, row_factory=dict_row,
|
||||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||||
|
||||
|
||||
def publish_due(conn, publisher: broker.Publisher) -> int:
|
||||
published = 0
|
||||
while not STOP.is_set():
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT id, job_id, attempt, payload FROM outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()
|
||||
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||||
(settings.DISPATCHER_BATCH,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return published
|
||||
for row in rows:
|
||||
publish_one(conn, publisher, row)
|
||||
published += 1
|
||||
log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
|
||||
if len(rows) < settings.DISPATCHER_BATCH:
|
||||
return published
|
||||
return published
|
||||
|
||||
|
||||
def publish_one(conn, publisher: broker.Publisher, row: dict) -> None:
|
||||
"""Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст
|
||||
уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу."""
|
||||
message = dict(row["payload"])
|
||||
job_id, attempt = message.get("job_id"), message.get("attempt")
|
||||
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send",
|
||||
"messaging.destination.name": pipeline.EXCHANGE,
|
||||
"messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY,
|
||||
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
|
||||
"messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]}
|
||||
parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate"))
|
||||
with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent,
|
||||
kind=SpanKind.PRODUCER, attributes=attributes,
|
||||
record_exception=False, set_status_on_exception=False) as span:
|
||||
carrier = telemetry.inject()
|
||||
if carrier.get("traceparent"):
|
||||
message["traceparent"] = carrier["traceparent"]
|
||||
message.pop("tracestate", None)
|
||||
if carrier.get("tracestate"):
|
||||
message["tracestate"] = carrier["tracestate"]
|
||||
faults.crash("dispatcher_crash_before_publish")
|
||||
try:
|
||||
publisher.publish(message)
|
||||
except broker.BrokerError as exc:
|
||||
telemetry.mark_error(span, type(exc).__name__)
|
||||
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"})
|
||||
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__,
|
||||
extra={"event": "outbox.publish_failed", "job.id": str(job_id),
|
||||
"error.type": type(exc).__name__})
|
||||
publisher.close()
|
||||
raise
|
||||
faults.crash("dispatcher_crash_after_confirm")
|
||||
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
|
||||
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"})
|
||||
|
||||
|
||||
def note_failure(conn) -> None:
|
||||
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
|
||||
try:
|
||||
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
|
||||
WHERE published_at IS NULL AND available_at <= now()""")
|
||||
except psycopg.Error:
|
||||
pass
|
||||
|
||||
|
||||
def touch_alive() -> None:
|
||||
try:
|
||||
Path(settings.ALIVE_FILE).touch()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def run() -> None:
|
||||
publisher = broker.Publisher()
|
||||
conn = None
|
||||
last_leases = last_reconcile = 0.0
|
||||
pause = 1.0
|
||||
while not STOP.is_set():
|
||||
try:
|
||||
if conn is None or conn.closed:
|
||||
conn = connect_db()
|
||||
conn.execute("LISTEN outbox")
|
||||
log.info("connected to PostgreSQL, listening for outbox notifications",
|
||||
extra={"event": "db.connected"})
|
||||
touch_alive()
|
||||
publish_due(conn, publisher)
|
||||
collect_stats(conn, publisher)
|
||||
now = time.monotonic()
|
||||
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
|
||||
last_leases = now
|
||||
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
|
||||
log.warning("recovered %s job(s) with expired lease", recovered,
|
||||
extra={"event": "lease.recovered", "count": recovered})
|
||||
if now - last_reconcile >= 30:
|
||||
last_reconcile = now
|
||||
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
|
||||
log.warning("re-published %s queued job(s) without a message in flight", lost,
|
||||
extra={"event": "outbox.republished", "count": lost})
|
||||
pause = 1.0
|
||||
publisher.pump()
|
||||
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
|
||||
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
|
||||
pass
|
||||
except broker.BrokerError:
|
||||
if conn is not None and not conn.closed:
|
||||
note_failure(conn)
|
||||
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause,
|
||||
extra={"event": "broker.unavailable"})
|
||||
STOP.wait(pause)
|
||||
pause = min(pause * 2, 15)
|
||||
except psycopg.Error as exc:
|
||||
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause,
|
||||
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
|
||||
try:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
except psycopg.Error:
|
||||
pass
|
||||
conn = None
|
||||
STOP.wait(pause)
|
||||
pause = min(pause * 2, 15)
|
||||
publisher.close()
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
log.info("stopped", extra={"event": "service.stopped"})
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not settings.BROKER_URL:
|
||||
raise SystemExit("BROKER_URL is required")
|
||||
secretstore.install_metrics(telemetry.meter)
|
||||
secretstore.store.start() # lab7: пароль БД dispatcher — из OpenBao (CN=dispatcher)
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, lambda *_: STOP.set())
|
||||
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
|
||||
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
|
||||
run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
# Runbooks алертов (lab6)
|
||||
|
||||
Правила — `infra/prometheus/rules/alerts.yml`, unit-тесты правил — `infra/prometheus/tests/alerts_test.yml`.
|
||||
Состояние алертов: http://127.0.0.1:9090/alerts или Grafana → Alerting → Alert rules (data source rules).
|
||||
Dashboard: Grafana http://127.0.0.1:3000 → Messenger → «Messenger — lab6 observability».
|
||||
Общий порядок: **метрика говорит «что и когда», трасса — «где именно», лог — «почему»**.
|
||||
|
||||
Ссылки на трассу из лога: Grafana → Explore → Loki → запись → поле `trace_id` → «Открыть трассу».
|
||||
Из спана в логи: Tempo → спан → «Logs for this span» (запрос по `trace_id`).
|
||||
|
||||
---
|
||||
|
||||
## MessengerHighErrorRate
|
||||
|
||||
**Что значит.** Больше 5% бизнес-запросов API (без `/health/*`) за минуту закончились 5xx, дольше минуты,
|
||||
при трафике > 0,2 запроса/с. 4xx (неверный пароль, чужой чат) сюда не входят.
|
||||
|
||||
**Влияние.** Пользователи получают 503/500, SLO-1 (99% без 5xx за час) расходует бюджет в 5+ раз быстрее нормы.
|
||||
|
||||
**Найти причину.**
|
||||
1. Какие маршруты и коды: dashboard «RPS по классу статуса» или
|
||||
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))`.
|
||||
503 — недоступна зависимость, 500 — ошибка кода.
|
||||
2. Медленная БД или медленный обработчик:
|
||||
- `histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))` и
|
||||
`db_client_connection_pending_requests` — запросы ждут соединение из пула → соединения заняты долгими запросами;
|
||||
- Tempo, TraceQL `{ resource.service.name = "messenger-api" && status = error }` → открыть трассу:
|
||||
если почти всё время server-спана занимает дочерний спан `INSERT`/`SELECT` (PostgreSQL) с ошибкой —
|
||||
проблема в БД; если время в `argon2id hash`/`inspect image` или «пустое» время самого server-спана — в обработчике;
|
||||
`redis GET` с ошибкой — Redis; `S3 PutObject` — SeaweedFS.
|
||||
3. Логи API: `{service_name="messenger-api"} | event="db.unavailable"` (поле `error_type`: `QueryCanceled` —
|
||||
statement_timeout 5 с, `PoolTimeout` — нет свободного соединения 3 с, `OperationalError` — БД недоступна);
|
||||
`| event="redis.unavailable"`, `| event="s3.unavailable"`.
|
||||
4. В PostgreSQL (если трассы указывают на БД): `docker compose exec postgres psql -U messenger -d messenger -c
|
||||
"SELECT pid, state, wait_event_type, wait_event, now()-query_start AS age, left(query,60) FROM pg_stat_activity
|
||||
WHERE datname='messenger' ORDER BY age DESC"` — блокировки (`wait_event_type = Lock`) и долгие запросы.
|
||||
|
||||
**Исправить.** Снять блокировку/долгий запрос (`pg_terminate_backend(pid)`), восстановить зависимость
|
||||
(`docker compose start postgres|redis|s3`). Не перезапускать API «на всякий случай»: причина не в нём.
|
||||
|
||||
**Проверить восстановление.** Доля 5xx на dashboard → 0, алерт становится inactive через ~1–2 минуты
|
||||
(окно rate 1m). Пример разбора: REPORT.md, инцидент 2.
|
||||
|
||||
---
|
||||
|
||||
## MessengerQueueLagGrowing
|
||||
|
||||
**Что значит.** Старейшее задание обработки вложения, которое уже можно брать (статус `queued`,
|
||||
`available_at` прошёл), ждёт больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно.
|
||||
|
||||
**Влияние.** Вложения остаются `queued`, клиенты не получают превью; SLO-3 (95% ready ≤ 30 с) нарушается.
|
||||
|
||||
**Найти причину.**
|
||||
1. `messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}` = 0 → worker не подключён
|
||||
(см. MessengerWorkerNoConsumers). > 0 → worker есть, но не успевает или падает на заданиях.
|
||||
2. Насыщение worker: `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` ≈ 1 и растёт
|
||||
`messenger_queue_depth` → нужно больше потоков/реплик worker (`WORKER_CONCURRENCY`, `--scale worker=2`).
|
||||
3. Ошибки обработки: `sum by (outcome) (rate(messenger_job_duration_seconds_count[5m]))` — много `retry`/`failed`;
|
||||
логи `{service_name="messenger-worker"} | event=~"job.retry|job.failed"` (поле `error_type`).
|
||||
4. Отличить от проблемы публикации: `messenger_outbox_oldest_age_seconds` большой → сообщения не доходят до
|
||||
брокера (dispatcher/RabbitMQ), а не worker; лог `{service_name="messenger-dispatcher"} | event="broker.unavailable"`.
|
||||
5. Трасса любой зависшей загрузки (по `trace_id` из лога API `event="http.request"` для POST attachments):
|
||||
есть `publish attachments`, но нет `process attachments.process.v1` → сообщение в брокере, его никто не взял.
|
||||
|
||||
**Исправить.** `docker compose start worker` (или `scripts/worker.sh start`); при насыщении — увеличить
|
||||
параллелизм; при массовых ошибках — устранить их причину, затем `ops.py replay` для failed.
|
||||
|
||||
**Проверить восстановление.** `messenger_jobs_backlog{status="queued"}` → 0, возраст → 0, вложения `ready`,
|
||||
алерт inactive через ~15–30 с. Пример: REPORT.md, инцидент 1.
|
||||
|
||||
---
|
||||
|
||||
## MessengerWorkerNoConsumers
|
||||
|
||||
**Что значит.** RabbitMQ дольше минуты не видит ни одного подписчика очереди `attachments.process.v1`.
|
||||
|
||||
**Найти причину.** `docker compose ps worker`; последние записи `{service_name="messenger-worker"}` в Loki
|
||||
(`event="service.stopped"` — остановлен штатно; `event="broker.unavailable"` — не может подключиться к брокеру;
|
||||
отсутствие записей и метрик `messenger_worker_capacity` — процесс не запущен).
|
||||
|
||||
**Исправить.** Запустить worker / восстановить RabbitMQ. Принятые задания не теряются (outbox + durable очередь).
|
||||
|
||||
---
|
||||
|
||||
## MessengerDbPoolSaturated
|
||||
|
||||
**Что значит.** На реплике API запросы дольше минуты ждут свободное соединение из пула (`DB_POOL_MAX`=10).
|
||||
|
||||
**Найти причину.** Почти всегда это следствие медленных запросов или блокировок в PostgreSQL, а не нехватки
|
||||
соединений: те же шаги, что в MessengerHighErrorRate (трассы с долгими DB-спанами, `pg_stat_activity`).
|
||||
Увеличивать пул имеет смысл, только если DB-спаны быстрые, а ожидание пула долгое при высоком RPS.
|
||||
|
||||
---
|
||||
|
||||
## TelemetryExportFailing
|
||||
|
||||
**Что значит.** Collector не может отправить данные в Tempo / Loki / Prometheus (метка `exporter`): очередь
|
||||
exporter дольше минуты заполнена больше чем на 5% (хранилище не принимает), либо пакеты уже теряются —
|
||||
у пакета истекло окно повторов 120 с (`send_failed`) или очередь переполнена (`enqueue_failed`). Окно повторов
|
||||
считается для пакета с первой попытки, время ожидания в очереди в него не входит: очередь (200 пакетов) —
|
||||
это и есть буфер на время недоступности.
|
||||
|
||||
**Влияние.** На пользователей не влияет. Телеметрия этого периода частично потеряна (сколько — счётчики
|
||||
`otelcol_exporter_send_failed_*`, `otelcol_exporter_enqueue_failed_*`).
|
||||
|
||||
**Найти причину.** `docker compose ps tempo loki prometheus`; `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`;
|
||||
логи Collector: `docker compose logs otel-collector --tail 50` (через Loki их не смотреть — это тот же конвейер).
|
||||
|
||||
**Исправить.** `docker compose start tempo loki prometheus`. Очередь Collector досылается сама (в пределах окна
|
||||
повторов). Проверить: `otelcol_exporter_queue_size` → 0, свежая трасса находится. Пример: REPORT.md, инцидент 3.
|
||||
|
||||
---
|
||||
|
||||
## TelemetryCollectorDown
|
||||
|
||||
**Что значит.** Prometheus не может забрать метрики Collector (`up{job="otel-collector"} == 0`).
|
||||
|
||||
**Влияние.** Приложение работает; SDK каждого процесса держит не больше 2048 спанов и 2048 записей логов.
|
||||
Потери считаются в самом процессе: пакет, который Collector не принял, — `otel_sdk_exporter_span_exported_total{error_type!=""}`
|
||||
(обычно `ConnectionError`), переполнение очереди — `otel_sdk_processor_span_processed_total{error_type="queue_full"}`. Метрик приложения за
|
||||
этот период нет (пропуск на графиках), cumulative-счётчики после восстановления продолжат расти без потери итогов.
|
||||
|
||||
**Исправить.** `docker compose start otel-collector`; проверить, что трассы и метрики снова приходят.
|
||||
@@ -0,0 +1,50 @@
|
||||
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def bad_request(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found(message: str) -> ApiError:
|
||||
return ApiError(404, "not_found", message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, "conflict", message)
|
||||
|
||||
|
||||
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
|
||||
return ApiError(503, "unavailable", message)
|
||||
|
||||
|
||||
def payload_too_large(message: str) -> ApiError:
|
||||
return ApiError(413, "payload_too_large", message)
|
||||
|
||||
|
||||
def unsupported_media_type(message: str) -> ApiError:
|
||||
return ApiError(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
def rate_limited(retry_after: int, message: str = "Too many attempts, retry later") -> ApiError:
|
||||
"""lab7: 429 + Retry-After (секунды до конца окна общего счётчика в Redis)."""
|
||||
return ApiError(429, "rate_limited", message, {"Retry-After": str(max(1, int(retry_after)))})
|
||||
|
||||
|
||||
def busy(retry_after: int = 1) -> ApiError:
|
||||
"""lab7: бюджет одновременных Argon2id-хешей реплики исчерпан — 503 + Retry-After, без очереди."""
|
||||
return ApiError(503, "unavailable", "Server is busy, retry later", {"Retry-After": str(max(1, int(retry_after)))})
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"rows": [
|
||||
{
|
||||
"m_kib": 19456,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 12.2,
|
||||
"verify_ms": 12.2,
|
||||
"max_logins_per_s_per_replica": 163.9
|
||||
},
|
||||
{
|
||||
"m_kib": 19456,
|
||||
"t": 3,
|
||||
"p": 1,
|
||||
"hash_ms": 18.3,
|
||||
"verify_ms": 18.5,
|
||||
"max_logins_per_s_per_replica": 108.1
|
||||
},
|
||||
{
|
||||
"m_kib": 32768,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 22.6,
|
||||
"verify_ms": 22.2,
|
||||
"max_logins_per_s_per_replica": 90.1
|
||||
},
|
||||
{
|
||||
"m_kib": 47104,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 34.2,
|
||||
"verify_ms": 34.8,
|
||||
"max_logins_per_s_per_replica": 57.5
|
||||
},
|
||||
{
|
||||
"m_kib": 65536,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 49.8,
|
||||
"verify_ms": 50.4,
|
||||
"max_logins_per_s_per_replica": 39.7
|
||||
},
|
||||
{
|
||||
"m_kib": 65536,
|
||||
"t": 3,
|
||||
"p": 1,
|
||||
"hash_ms": 75.3,
|
||||
"verify_ms": 75.2,
|
||||
"max_logins_per_s_per_replica": 26.6
|
||||
}
|
||||
],
|
||||
"maxrss_mib": 96.1,
|
||||
"current": [
|
||||
32768,
|
||||
2,
|
||||
1
|
||||
],
|
||||
"max_concurrency": 2,
|
||||
"queue_s": 2
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
# argon2-bench: lab7: Argon2id cost inside the api container for candidate parameters
|
||||
# date: 2026-11-07 19:08:27, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/argon2_bench.py
|
||||
# exit code: 0, 3.9 s
|
||||
|
||||
m (KiB) t p hash ms verify ms logins/s per replica at concurrency 2 RAM for hashing
|
||||
19456 2 1 12.2 12.2 163.9 38 MiB
|
||||
19456 3 1 18.3 18.5 108.1 38 MiB
|
||||
32768 2 1 22.6 22.2 90.1 64 MiB <- current
|
||||
47104 2 1 34.2 34.8 57.5 92 MiB
|
||||
65536 2 1 49.8 50.4 39.7 128 MiB
|
||||
65536 3 1 75.3 75.2 26.6 128 MiB
|
||||
process max RSS during the bench: 96.1 MiB
|
||||
saved evidence/argon2-bench.json
|
||||
@@ -0,0 +1,96 @@
|
||||
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||
# date: 2026-11-07 19:40:25, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_attachments.py
|
||||
# exit code: 0, 12.8 s
|
||||
|
||||
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||
exactly 10485760 bytes (10 MiB) -> 202, then ready
|
||||
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||
PASS: the 10 MiB limit is exact
|
||||
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||
PASS: oversized streamed body is cut off, the API keeps working
|
||||
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 54186d3d
|
||||
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||
PASS: pixel limit is checked before decompression
|
||||
PNG cut in half (55908 bytes, image/png) -> 415
|
||||
PNG without IEND (111805 bytes, image/png) -> 415
|
||||
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||
PNG signature only (8 bytes, image/png) -> 415
|
||||
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
|
||||
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||
random bytes (1600 bytes, image/png) -> 415
|
||||
plain text (28 bytes, image/jpeg) -> 415
|
||||
empty body -> 400
|
||||
PASS: damaged and foreign files are rejected with 415, never 500
|
||||
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
|
||||
IEND with data (70 bytes, valid CRCs) -> 415
|
||||
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||
PNG bytes sent as image/jpeg -> 415
|
||||
JPEG bytes sent as image/png -> 415
|
||||
GIF bytes sent as image/png -> 415
|
||||
PNG sent as image/gif -> 415
|
||||
PNG sent as text/html -> 415
|
||||
PNG sent as application/octet-stream -> 415
|
||||
PNG sent as multipart/form-data -> 415
|
||||
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||
no filename -> 400
|
||||
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||
PASS: filename never becomes a path; dangerous names are refused
|
||||
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
|
||||
PASS: only chat members can see an attachment
|
||||
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||
attachment of another chat (author is a member of both) -> 404
|
||||
own + foreign attachment -> 404
|
||||
unknown attachment_id -> 404
|
||||
outsider referencing an attachment of the chat -> 404
|
||||
duplicate ids -> 400
|
||||
11 ids -> 400
|
||||
bad UUID -> 400
|
||||
empty text without attachments -> 400
|
||||
whitespace text without attachments -> 400
|
||||
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||
PASS: Idempotency-Key covers attachment_ids
|
||||
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||
PASS: an interrupted upload is dropped
|
||||
12 passed, 0 failed
|
||||
@@ -0,0 +1,18 @@
|
||||
# balancing: balancing, shared sessions/data/files, X-Instance-Id
|
||||
# date: 2026-11-07 19:43:46, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_balancing.py
|
||||
# exit code: 0, 3.5 s
|
||||
|
||||
20 x GET /users/me with one token -> replicas: {'f9e5ac61cde6': 10, '708b4b92f569': 10}
|
||||
PASS: one session works on every replica, no second login (sessions are shared in Redis)
|
||||
message written by 708b4b92f569; read back by ['708b4b92f569', 'f9e5ac61cde6']
|
||||
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
|
||||
logout on f9e5ac61cde6; old token then rejected by ['708b4b92f569', 'f9e5ac61cde6']
|
||||
PASS: logout on one replica revokes the session on all of them
|
||||
first POST on f9e5ac61cde6 -> 201 b67e1df0-7002-437c-ba25-1ca953b58180; same key on 708b4b92f569 -> 201 b67e1df0-7002-437c-ba25-1ca953b58180
|
||||
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
|
||||
file uploaded via f9e5ac61cde6; downloaded byte for byte via ['708b4b92f569', 'f9e5ac61cde6']
|
||||
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
|
||||
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running replica ['708b4b92f569', 'f9e5ac61cde6']; api containers -> aliases: {'0c7893e1043d': '708b4b92f569', 'b7f37f051ba1': 'f9e5ac61cde6'}
|
||||
PASS: X-Instance-Id is in every response and points to a real container
|
||||
6 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# check: offline contract check (make check)
|
||||
# date: 2026-11-07 19:07:13, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe scripts/check_contract.py
|
||||
# exit code: 0, 0.2 s
|
||||
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,34 @@
|
||||
# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning
|
||||
# date: 2026-11-07 19:12:52, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/check_configs.py
|
||||
# exit code: 0, 5.7 s
|
||||
|
||||
--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0
|
||||
|
||||
--- promtool check config (infra/prometheus/prometheus.yml): exit 0
|
||||
|
||||
Checking /etc/prometheus/rules/alerts.yml
|
||||
SUCCESS: 6 rules found
|
||||
|
||||
Checking /etc/prometheus/rules/recording.yml
|
||||
SUCCESS: 10 rules found
|
||||
--- promtool check rules (recording + alerts): exit 0
|
||||
Checking /etc/prometheus/rules/alerts.yml
|
||||
SUCCESS: 6 rules found
|
||||
|
||||
Checking /etc/prometheus/rules/recording.yml
|
||||
SUCCESS: 10 rules found
|
||||
--- promtool check web-config (lab7: TLS, client certs, allowed SANs): exit 0
|
||||
/etc/prometheus/web.yml SUCCESS
|
||||
--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0
|
||||
SUCCESS
|
||||
--- loki -verify-config (infra/loki/loki.yaml): exit 0
|
||||
|
||||
--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0
|
||||
|
||||
--- Grafana 12.4.14: database ok
|
||||
datasource prometheus (prometheus): OK Successfully queried the Prometheus API.
|
||||
datasource loki (loki): OK Data source successfully connected.
|
||||
datasource tempo (tempo): OK Data source is working
|
||||
dashboard 'Messenger — lab6 observability' in folder 'Messenger': 34 panels, provisioned=True, datasources ['loki', 'prometheus', 'tempo']
|
||||
PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana provisioned 3 healthy datasources and the dashboard without manual steps
|
||||
@@ -0,0 +1,47 @@
|
||||
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
|
||||
# date: 2026-11-07 19:42:46, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_consistency.py
|
||||
# exit code: 0, 131.5 s
|
||||
|
||||
declared 9 000 000 bytes, sent 211817, closed the connection
|
||||
declared 9 000 000 bytes, sent 3111817, closed the connection
|
||||
rows in attachments: 481 -> 481; objects in bucket: 1886 -> 1886
|
||||
temporary upload files in api containers: {'b7f37f051ba1': 0, '0c7893e1043d': 0} (removed when the request ends)
|
||||
api containers have no volumes: the original is never stored locally
|
||||
PASS: an interrupted upload leaves no row, no object and no local file
|
||||
crash after S3 PUT: row aa892679 state=uploading, object staging/a6d6cacccc714e1683e0cbf58f6c3bb8 exists in S3
|
||||
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
|
||||
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1891, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.698} -> fresh row and object are kept (the upload might still be running)
|
||||
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1891, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.695} -> nothing deleted
|
||||
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1890, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.691} -> row and object deleted
|
||||
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1890, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.599} -> idempotent
|
||||
the ready attachment of the same chat is untouched
|
||||
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
|
||||
object staging/71f7fcf571ca43e18de2aa82aac94d8e put into the bucket without a row in PostgreSQL
|
||||
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1895, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.689} -> fresh object kept
|
||||
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1895, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.793} -> reported, kept
|
||||
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1895, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 2, "seconds": 0.678} -> orphan deleted
|
||||
ready attachments older than the grace period are untouched (they have rows)
|
||||
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=3 and keeps the row; download -> 503 (not 500)
|
||||
PASS: orphans are found and deleted only after the grace period; missing objects are reported
|
||||
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
|
||||
upload -> 503 unavailable in 781 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose start s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 202 (14.7 s after the old file became readable)
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1905, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 3, "seconds": 0.695}
|
||||
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
--- S3 frozen (no answer, timeouts): docker compose pause s3
|
||||
upload -> 503 unavailable in 8703 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose unpause s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 202 (0.3 s after the old file became readable)
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1914, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 3, "seconds": 0.685}
|
||||
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
5 passed, 0 failed
|
||||
@@ -0,0 +1,36 @@
|
||||
# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels
|
||||
# date: 2026-11-07 19:14:22, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/check_dashboard.py
|
||||
# exit code: 0, 2.7 s
|
||||
|
||||
PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident)
|
||||
empty now: CPU контейнеров / лимит
|
||||
empty now: Collector: потери и отказы
|
||||
empty now: SDK приложений: экспорт и потери
|
||||
empty now: alerts.yml: MessengerDbPoolSaturated
|
||||
empty now: alerts.yml: MessengerHighErrorRate
|
||||
empty now: alerts.yml: MessengerQueueLagGrowing
|
||||
empty now: alerts.yml: MessengerWorkerNoConsumers
|
||||
empty now: alerts.yml: TelemetryCollectorDown
|
||||
empty now: alerts.yml: TelemetryExportFailing
|
||||
empty now: Алерты Prometheus (pending/firing)
|
||||
empty now: Память: RSS процесса и контейнер
|
||||
LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error"
|
||||
TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 5 trace(s)
|
||||
|
||||
data: http_server_request_duration_seconds_count{job="messenger-api"} 54 series <- HTTP requests/duration (3)
|
||||
data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3)
|
||||
data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4)
|
||||
data: messenger_queue_depth 2 series <- queue depth (4)
|
||||
data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4)
|
||||
data: db_client_connection_count 6 series <- DB pool used/idle (4)
|
||||
data: db_client_connection_pending_requests 3 series <- DB pool waiting (4)
|
||||
data: messenger_worker_busy 1 series <- worker busy (4)
|
||||
data: messenger_worker_capacity 1 series <- worker capacity (4)
|
||||
data: process_cpu_time_seconds_total 16 series <- process CPU (4)
|
||||
data: process_memory_usage_bytes 8 series <- process RAM (4)
|
||||
data: otelcol_exporter_queue_size 3 series <- collector queue (4)
|
||||
|
||||
labels used by application metrics: __name__, bao_role, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, error_type, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, messenger_rate_limit, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
|
||||
http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready']
|
||||
PASS: every dashboard/rule query runs, key signals have data, no forbidden labels
|
||||
@@ -0,0 +1,21 @@
|
||||
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
|
||||
# date: 2026-11-07 19:44:08, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_db.py
|
||||
# exit code: 0, 11.1 s
|
||||
|
||||
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 389 -> 390
|
||||
PASS: chat row was inserted and then rolled back together with members
|
||||
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
|
||||
PASS: the database itself rejects a message from a non-member
|
||||
stored password: $argon2id$v=19$m=32768,t=2,p=1$... (length 97)
|
||||
PASS: password is stored as Argon2id hash
|
||||
unique constraint on users: UNIQUE (username)
|
||||
PASS: username is unique in the database
|
||||
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; running the migrate service once more
|
||||
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; log: Migrations applied: none (schema is up to date)
|
||||
PASS: migrations are applied once, a repeated run does nothing
|
||||
api replicas that checked the schema at start: 11; api log lines 'Migrations applied': 0; migrate service: Exited (0) 3 minutes ago
|
||||
PASS: only the migrate service applies migrations, replicas wait for the schema
|
||||
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
|
||||
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
|
||||
7 passed, 0 failed
|
||||
@@ -0,0 +1,112 @@
|
||||
=== Data volume
|
||||
messages_total | messages_in_big_chat
|
||||
----------------+----------------------
|
||||
141938 | 15000
|
||||
|
||||
|
||||
=== History index
|
||||
indexdef
|
||||
----------------------------------------------------------------------------------------
|
||||
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
|
||||
|
||||
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.007..0.018 rows=51 loops=1)
|
||||
Buffers: shared hit=54
|
||||
-> Index Scan using messages_history on messages (actual time=0.007..0.015 rows=51 loops=1)
|
||||
Index Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
|
||||
Buffers: shared hit=54
|
||||
Planning:
|
||||
Buffers: shared hit=7
|
||||
Planning Time: 0.066 ms
|
||||
Execution Time: 0.026 ms
|
||||
|
||||
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.027..0.041 rows=51 loops=1)
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using messages_history on messages (actual time=0.026..0.036 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'cc573555-8177-419f-a1b0-2b402c9b73d3'::uuid)))
|
||||
Buffers: shared hit=4
|
||||
Planning Time: 0.084 ms
|
||||
Execution Time: 0.098 ms
|
||||
|
||||
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.021..0.177 rows=51 loops=1)
|
||||
Buffers: shared hit=58
|
||||
-> Index Scan using messages_history on messages m (actual time=0.021..0.174 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'cc573555-8177-419f-a1b0-2b402c9b73d3'::uuid)))
|
||||
Buffers: shared hit=58
|
||||
SubPlan 1
|
||||
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
|
||||
Sort Key: a."position"
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Recheck Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Index Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
Planning:
|
||||
Buffers: shared hit=63
|
||||
Planning Time: 0.164 ms
|
||||
Execution Time: 0.219 ms
|
||||
|
||||
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
QUERY PLAN
|
||||
-------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=3.398..3.401 rows=51 loops=1)
|
||||
Buffers: shared hit=2553
|
||||
-> Sort (actual time=3.012..3.217 rows=10051 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: quicksort Memory: 2377kB
|
||||
Buffers: shared hit=2553
|
||||
-> Bitmap Heap Scan on messages (actual time=0.608..1.680 rows=15000 loops=1)
|
||||
Recheck Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
|
||||
Heap Blocks: exact=2359
|
||||
Buffers: shared hit=2553
|
||||
-> Bitmap Index Scan on messages_history (actual time=0.458..0.458 rows=15000 loops=1)
|
||||
Index Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
|
||||
Buffers: shared hit=194
|
||||
Planning Time: 0.036 ms
|
||||
Execution Time: 3.484 ms
|
||||
|
||||
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
QUERY PLAN
|
||||
-----------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=1.636..1.639 rows=51 loops=1)
|
||||
Buffers: shared hit=2375
|
||||
-> Sort (actual time=1.635..1.637 rows=51 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 31kB
|
||||
Buffers: shared hit=2375
|
||||
-> Bitmap Heap Scan on messages (actual time=1.034..1.289 rows=5000 loops=1)
|
||||
Recheck Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
|
||||
Filter: (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'cc573555-8177-419f-a1b0-2b402c9b73d3'::uuid))
|
||||
Rows Removed by Filter: 10000
|
||||
Heap Blocks: exact=2359
|
||||
Buffers: shared hit=2375
|
||||
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.303..0.303 rows=15000 loops=1)
|
||||
Index Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
|
||||
Buffers: shared hit=16
|
||||
Planning:
|
||||
Buffers: shared hit=4
|
||||
Planning Time: 0.099 ms
|
||||
Execution Time: 1.665 ms
|
||||
|
||||
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
explain_users_left
|
||||
--------------------
|
||||
0
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
|
||||
# date: 2026-11-07 19:44:13, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/explain.py
|
||||
# exit code: 0, 0.8 s
|
||||
|
||||
=== Data volume
|
||||
=== History index
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
Buffers: shared hit=54
|
||||
-> Index Scan using messages_history on messages (actual time=0.007..0.015 rows=51 loops=1)
|
||||
Buffers: shared hit=54
|
||||
Buffers: shared hit=7
|
||||
Execution Time: 0.026 ms
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using messages_history on messages (actual time=0.026..0.036 rows=51 loops=1)
|
||||
Buffers: shared hit=4
|
||||
Execution Time: 0.098 ms
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
Buffers: shared hit=58
|
||||
-> Index Scan using messages_history on messages m (actual time=0.021..0.174 rows=51 loops=1)
|
||||
Buffers: shared hit=58
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
Buffers: shared hit=63
|
||||
Execution Time: 0.219 ms
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
Buffers: shared hit=2553
|
||||
Buffers: shared hit=2553
|
||||
-> Bitmap Heap Scan on messages (actual time=0.608..1.680 rows=15000 loops=1)
|
||||
Buffers: shared hit=2553
|
||||
-> Bitmap Index Scan on messages_history (actual time=0.458..0.458 rows=15000 loops=1)
|
||||
Buffers: shared hit=194
|
||||
Execution Time: 3.484 ms
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
Buffers: shared hit=2375
|
||||
Buffers: shared hit=2375
|
||||
-> Bitmap Heap Scan on messages (actual time=1.034..1.289 rows=5000 loops=1)
|
||||
Buffers: shared hit=2375
|
||||
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.303..0.303 rows=15000 loops=1)
|
||||
Buffers: shared hit=16
|
||||
Buffers: shared hit=4
|
||||
Execution Time: 1.665 ms
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
PASS: full plans saved to evidence\explain-history.txt
|
||||
@@ -0,0 +1,33 @@
|
||||
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
|
||||
# date: 2026-11-07 19:45:00, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/run_failover.py
|
||||
# exit code: 0, 46.3 s
|
||||
|
||||
hook: scripts\stop-one.cmd; api replicas: ['0c7893e1043d', 'b7f37f051ba1']
|
||||
=== run 1: api containers before
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b7f37f051ba1 Up 49 seconds (healthy)
|
||||
api 0c7893e1043d Up 49 seconds (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 5.453}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b7f37f051ba1 Up About a minute (healthy)
|
||||
api 0c7893e1043d Exited (0) 11 seconds ago
|
||||
the hook stopped 0c7893e1043d (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start 0c7893e1043d -> healthy and back in HAProxy
|
||||
=== run 2: api containers before
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b7f37f051ba1 Up About a minute (healthy)
|
||||
api 0c7893e1043d Up 5 seconds (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 5.313}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b7f37f051ba1 Exited (0) 11 seconds ago
|
||||
api 0c7893e1043d Up 23 seconds (healthy)
|
||||
the hook stopped b7f37f051ba1 (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start b7f37f051ba1 -> healthy and back in HAProxy
|
||||
PASS: public failover test passed 2 times; every api replica was stopped once and restored
|
||||
@@ -0,0 +1,29 @@
|
||||
# failures: Redis and PostgreSQL outages via the balancer
|
||||
# date: 2026-11-07 19:46:03, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_failures.py
|
||||
# exit code: 0, 32.4 s
|
||||
|
||||
stopping redis
|
||||
/health/live -> 200, /health/ready -> 503 (0 ms)
|
||||
valid token: GET /users/me -> 503 unavailable (16 ms), session cannot be checked
|
||||
unknown token -> 503, not 200: nobody is trusted without a check
|
||||
no token -> 401
|
||||
login -> 503: there is nowhere to store the session
|
||||
registration -> 201: it needs only PostgreSQL
|
||||
starting redis
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: Redis outage gives 503 and does not let anyone in without a session check
|
||||
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
|
||||
right after the crash: GET /users/me -> 503
|
||||
starting redis
|
||||
after start: the session issued before the crash still works (AOF, appendfsync always)
|
||||
PASS: sessions survive a Redis crash, not only a clean restart
|
||||
stopping postgres
|
||||
/health/live -> 200, /health/ready -> 503 (1016 ms)
|
||||
valid token: users/me, chats, history, new message -> 503 unavailable, no details
|
||||
no token -> 401, unknown token -> 401: anonymous users are not let in
|
||||
login -> 503, registration -> 503
|
||||
starting postgres
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
|
||||
3 passed, 0 failed
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
# hardening: lab7: no privileged/host net/docker.sock, caps dropped, no-new-privileges, read-only, non-root, limits, ports
|
||||
# date: 2026-11-07 19:07:19, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/check_hardening.py --runtime
|
||||
# exit code: 0, 0.8 s
|
||||
|
||||
EXCEPTION otel-collector network network_mode service:otel-gate — общий namespace с mTLS-шлюзом, Collector слушает только 127.0.0.1
|
||||
EXCEPTION rabbitmq read_only Erlang/RabbitMQ пишет cookie, схему и plugins-expand в /var/lib/rabbitmq и служебные файлы в /etc/rabbitmq при старте; данные — в volume, корень остав
|
||||
EXCEPTION telemetry-init cap_add одноразовый chown volumes Loki/Tempo для uid 10001: только CAP_CHOWN, без сети (network_mode none), сразу завершается
|
||||
EXCEPTION telemetry-init non_root chown требует uid 0 (+CAP_CHOWN)
|
||||
static: {'PASS': 183, 'EXCEPTION': 4, 'FAIL': 0}
|
||||
runtime api user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
|
||||
runtime api user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
|
||||
runtime dispatcher user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=134217728 pids=256 ports=[]
|
||||
runtime grafana user=472 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=402653184 pids=256 ports=['3000/tcp']
|
||||
runtime janitor user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=201326592 pids=256 ports=[]
|
||||
runtime lb user=haproxy ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=134217728 pids=256 ports=['8404/tcp', '8443/tcp']
|
||||
runtime loki user=10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=['3100/tcp']
|
||||
runtime openbao user=100 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=268435456 pids=256 ports=['8200/tcp']
|
||||
runtime otel-collector user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=402653184 pids=256 ports=[]
|
||||
runtime otel-gate user=haproxy ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=67108864 pids=256 ports=[]
|
||||
runtime postgres user=70:70 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
|
||||
runtime prometheus user=nobody ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=['9090/tcp']
|
||||
runtime rabbitmq user=rabbitmq ro=False priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=512 ports=[]
|
||||
runtime redis user=999:1000 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=134217728 pids=256 ports=[]
|
||||
runtime s3 user=1000:1000 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
|
||||
runtime tempo user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=['3200/tcp']
|
||||
runtime worker user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=1610612736 pids=128 ports=[]
|
||||
PASS: no privileged/host network/docker.sock; caps dropped; no-new-privileges; read-only roots; non-root; CPU/RAM/PID limits; only 127.0.0.1 ports — exceptions are explained
|
||||
@@ -0,0 +1,22 @@
|
||||
# idempotency: Idempotency-Key, parallel same key via replicas
|
||||
# date: 2026-11-07 19:43:50, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_idempotency.py
|
||||
# exit code: 0, 4.0 s
|
||||
|
||||
first -> 201; same JSON written differently -> 201; plain retry -> 201
|
||||
PASS: a retry with the same key returns the original 201 and message, no duplicate
|
||||
20 parallel POSTs with one key: statuses {201: 20}, replicas {'f9e5ac61cde6': 10, '708b4b92f569': 10}, different bodies in answers: 1, messages in history: 1
|
||||
PASS: parallel requests with one key through different replicas create exactly one message
|
||||
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
|
||||
PASS: the first body wins, the same body gets 201, the other body gets 409
|
||||
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
|
||||
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
|
||||
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
|
||||
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
|
||||
key used by a 400 and by a 404 first, then valid request -> 201
|
||||
PASS: rejected requests (400, 404) do not store a result for the key
|
||||
same key and body by an outsider -> 404; by alice after logout -> 401
|
||||
PASS: a retry checks the session and chat access again, the saved answer is not leaked
|
||||
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
|
||||
PASS: an unconfirmed POST can be safely retried with the same key
|
||||
8 passed, 0 failed
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"scenario": "collector-down",
|
||||
"date": "2026-11-07 19:27:10",
|
||||
"outage_s": 90,
|
||||
"baseline": {
|
||||
"requests": 796,
|
||||
"codes": {
|
||||
"201": 796
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 53.1,
|
||||
"p95_ms_successful": 32.0
|
||||
},
|
||||
"during": {
|
||||
"requests": 5298,
|
||||
"codes": {
|
||||
"201": 5298
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 53.3,
|
||||
"p95_ms_successful": 32.0
|
||||
},
|
||||
"api_mib_before": 831.6,
|
||||
"api_mib_max": 866.4,
|
||||
"sdk_lost_spans_api": {
|
||||
"processor_queue_full": 0,
|
||||
"exporter_failed": 19456
|
||||
},
|
||||
"sdk_exported_ok_after_restart": 5554,
|
||||
"after": {
|
||||
"requests": 1347,
|
||||
"codes": {
|
||||
"201": 1347
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 53.7,
|
||||
"p95_ms_successful": 32.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted
|
||||
# date: 2026-11-07 19:29:44, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/incidents.py collector-down
|
||||
# exit code: 0, 154.3 s
|
||||
|
||||
=== collector-down
|
||||
[ 24.6s] baseline: {'requests': 796, 'codes': {'201': 796}, 'success_ratio': 1.0, 'success_rps': 53.1, 'p95_ms_successful': 32.0}, api containers 831.6 MiB
|
||||
[ 25.1s] docker compose stop otel-collector
|
||||
[ 29.1s] {'t': 0.5, 'api_mib': 788.1}
|
||||
[ 43.1s] {'t': 14.5, 'api_mib': 850.0}
|
||||
[ 57.1s] {'t': 28.5, 'api_mib': 862.5}
|
||||
[ 71.1s] {'t': 42.5, 'api_mib': 866.4}
|
||||
[ 85.7s] {'t': 56.4, 'api_mib': 857.3}
|
||||
[ 99.7s] {'t': 71.1, 'api_mib': 846.4}
|
||||
[ 113.7s] {'t': 85.0, 'api_mib': 829.6}
|
||||
[ 124.1s] docker compose start otel-collector
|
||||
[ 154.2s] API during collector outage: {'requests': 5298, 'codes': {'201': 5298}, 'success_ratio': 1.0, 'success_rps': 53.3, 'p95_ms_successful': 32.0}; api memory 831.6 -> max 866.4 MiB; SDK lost spans (api): {'processor_queue_full': 0, 'exporter_failed': 19456}; exported OK in the last minute after restart: 5554
|
||||
[ 154.2s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-collector-down.json
|
||||
PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"scenario": "db-slow",
|
||||
"date": "2026-11-07 19:17:54",
|
||||
"lock_seconds": 210,
|
||||
"baseline": {
|
||||
"requests": 2042,
|
||||
"codes": {
|
||||
"201": 2042
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 102.1,
|
||||
"p95_ms_successful": 32.0,
|
||||
"server_p95_s": 0.0236
|
||||
},
|
||||
"firing_after_lock_s": 122.5,
|
||||
"diagnosis": {
|
||||
"http_5xx_ratio_1m": 1.0,
|
||||
"http_p95_messages_s": 7.375,
|
||||
"pool_used": 8.0,
|
||||
"pool_wait_p95_s": 0.001,
|
||||
"status_codes_1m": {
|
||||
"201": 0.0,
|
||||
"200": 0.0,
|
||||
"400": 0.0,
|
||||
"409": 0.0,
|
||||
"404": 0.0,
|
||||
"503": 1.44
|
||||
},
|
||||
"reads_during_lock_ms": {
|
||||
"GET /users/me + GET /chats, avg per request": 15.6,
|
||||
"max": 31.5
|
||||
},
|
||||
"api_log_db_unavailable_by_error_type": {
|
||||
"QueryCanceled": 184
|
||||
},
|
||||
"error_trace": {
|
||||
"trace_id": "50418f127c336af7f2e4dc0190adf7ea",
|
||||
"server_ms": 5010.0,
|
||||
"slowest_db_span": "INSERT",
|
||||
"slowest_db_ms": 5001.6,
|
||||
"db_span_status": "ERROR",
|
||||
"db_share": 0.998,
|
||||
"tree": [
|
||||
"- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5010.0 ms) ERROR http.response.status_code=503 error.type=503",
|
||||
" - [api] redis GET (CLIENT, 1.5 ms)",
|
||||
" - [api] SELECT (CLIENT, 0.6 ms)",
|
||||
" - [api] INSERT (CLIENT, 5001.6 ms) ERROR"
|
||||
]
|
||||
},
|
||||
"handler_cpu_contrast": {
|
||||
"trace_id": "2e774b9d5912ba768f3e9881f3c41e80",
|
||||
"server_ms": 47.0,
|
||||
"argon2id_ms": 32.2,
|
||||
"db_ms_total": 6.9,
|
||||
"tree": [
|
||||
"- [api] POST /api/v1/users (SERVER, 47.0 ms) http.response.status_code=201",
|
||||
" - [api] redis INCR (CLIENT, 6.2 ms)",
|
||||
" - [api] argon2id hash (INTERNAL, 32.2 ms)",
|
||||
" - [api] INSERT (CLIENT, 0.7 ms)"
|
||||
]
|
||||
}
|
||||
},
|
||||
"during_lock": {
|
||||
"requests": 328,
|
||||
"codes": {
|
||||
"503": 320,
|
||||
"201": 8
|
||||
},
|
||||
"success_ratio": 0.0244,
|
||||
"success_rps": 0.0,
|
||||
"p95_ms_successful": 2031.0
|
||||
},
|
||||
"resolved_after_unlock_s": 35.1,
|
||||
"after": {
|
||||
"requests": 3086,
|
||||
"codes": {
|
||||
"201": 3086
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 102.4,
|
||||
"p95_ms_successful": 32.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved
|
||||
# date: 2026-11-07 19:22:25, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/incidents.py db-slow
|
||||
# exit code: 0, 271.0 s
|
||||
|
||||
=== db-slow
|
||||
[ 0.1s] load: 8 threads POST /messages
|
||||
[ 25.2s] baseline: {'requests': 2042, 'codes': {'201': 2042}, 'success_ratio': 1.0, 'success_rps': 102.1, 'p95_ms_successful': 32.0, 'server_p95_s': 0.0236}
|
||||
[ 25.2s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait)
|
||||
[ 25.3s] 5xx ratio=0.0 p95=0.024 s pool used=0.0 waiting=0.0
|
||||
[ 30.4s] 5xx ratio=0.0 p95=0.024 s pool used=0.0 waiting=0.0
|
||||
[ 35.5s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
|
||||
[ 40.6s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
|
||||
[ 45.7s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
|
||||
[ 50.7s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
|
||||
[ 55.8s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
|
||||
[ 60.9s] 5xx ratio=0.008 p95=0.024 s pool used=8.0 waiting=0.0
|
||||
[ 66.0s] 5xx ratio=0.008 p95=0.025 s pool used=8.0 waiting=0.0
|
||||
[ 71.1s] 5xx ratio=0.008 p95=0.025 s pool used=8.0 waiting=0.0
|
||||
[ 76.3s] 5xx ratio=0.016 p95=6.67 s pool used=8.0 waiting=0.0
|
||||
[ 81.4s] 5xx ratio=0.016 p95=6.737 s pool used=8.0 waiting=0.0
|
||||
[ 86.5s] 5xx ratio=0.016 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 91.6s] 5xx ratio=0.14 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 96.7s] 5xx ratio=0.14 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 101.8s] 5xx ratio=0.14 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 106.9s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 112.0s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 117.1s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 122.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 127.4s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 132.5s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||
[ 137.6s] 5xx ratio=1.0 p95=7.375 s pool used=2.0 waiting=0.0
|
||||
[ 142.7s] 5xx ratio=1.0 p95=7.375 s pool used=2.0 waiting=0.0
|
||||
[ 147.7s] ALERT MessengerHighErrorRate firing (122.5 s after lock)
|
||||
[ 152.2s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "400": 0.0, "409": 0.0, "404": 0.0, "503": 1.44}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 15.6, "max": 31.5}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 184}}
|
||||
[ 152.2s] error trace (DB): {"trace_id": "50418f127c336af7f2e4dc0190adf7ea", "server_ms": 5010.0, "slowest_db_span": "INSERT", "slowest_db_ms": 5001.6, "db_span_status": "ERROR", "db_share": 0.998}
|
||||
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5010.0 ms) ERROR http.response.status_code=503 error.type=503
|
||||
- [api] redis GET (CLIENT, 1.5 ms)
|
||||
- [api] SELECT (CLIENT, 0.6 ms)
|
||||
- [api] INSERT (CLIENT, 5001.6 ms) ERROR
|
||||
[ 152.2s] contrast, slow handler (CPU): {"trace_id": "2e774b9d5912ba768f3e9881f3c41e80", "server_ms": 47.0, "argon2id_ms": 32.2, "db_ms_total": 6.9}
|
||||
- [api] POST /api/v1/users (SERVER, 47.0 ms) http.response.status_code=201
|
||||
- [api] redis INCR (CLIENT, 6.2 ms)
|
||||
- [api] argon2id hash (INTERNAL, 32.2 ms)
|
||||
- [api] INSERT (CLIENT, 0.7 ms)
|
||||
[ 235.7s] lock released (psql exit 0), 210.5 s after LOCK
|
||||
[ 235.7s] ... MessengerHighErrorRate: firing (0 s)
|
||||
[ 240.7s] ... MessengerHighErrorRate: firing (5 s)
|
||||
[ 245.8s] ... MessengerHighErrorRate: firing (10 s)
|
||||
[ 250.8s] ... MessengerHighErrorRate: firing (15 s)
|
||||
[ 255.8s] ... MessengerHighErrorRate: firing (20 s)
|
||||
[ 260.8s] ... MessengerHighErrorRate: firing (25 s)
|
||||
[ 265.8s] ... MessengerHighErrorRate: firing (30 s)
|
||||
[ 270.8s] ALERT MessengerHighErrorRate resolved 35.1 s after unlock
|
||||
[ 270.8s] load: during lock {'requests': 328, 'codes': {'503': 320, '201': 8}, 'success_ratio': 0.0244, 'success_rps': 0.0, 'p95_ms_successful': 2031.0}; after {'requests': 3086, 'codes': {'201': 3086}, 'success_ratio': 1.0, 'success_rps': 102.4, 'p95_ms_successful': 32.0}
|
||||
[ 270.9s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-db-slow.json
|
||||
PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,273 @@
|
||||
{
|
||||
"scenario": "telemetry-down",
|
||||
"date": "2026-11-07 19:22:25",
|
||||
"stopped": [
|
||||
"tempo",
|
||||
"loki"
|
||||
],
|
||||
"outage_s": 150,
|
||||
"baseline": {
|
||||
"requests": 793,
|
||||
"codes": {
|
||||
"201": 793
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 52.9,
|
||||
"p95_ms_successful": 32.0
|
||||
},
|
||||
"collector_heap_before_mib": 32.3,
|
||||
"collector_container_before_mib": 76.9,
|
||||
"during": {
|
||||
"requests": 8486,
|
||||
"codes": {
|
||||
"201": 8486
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 52.9,
|
||||
"p95_ms_successful": 32.0
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"t": 0.0,
|
||||
"collector_heap_mib": 51.5,
|
||||
"collector_rss_mib": 237.8,
|
||||
"collector_container_mib": 76.9,
|
||||
"queue": {
|
||||
"otlp_http/loki": 16.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 7.0
|
||||
},
|
||||
"alert": "inactive"
|
||||
},
|
||||
{
|
||||
"t": 12.1,
|
||||
"collector_heap_mib": 60.3,
|
||||
"collector_rss_mib": 237.8,
|
||||
"collector_container_mib": 76.9,
|
||||
"queue": {
|
||||
"otlp_http/loki": 16.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 7.0
|
||||
},
|
||||
"alert": "inactive"
|
||||
},
|
||||
{
|
||||
"t": 24.8,
|
||||
"collector_heap_mib": 44.3,
|
||||
"collector_rss_mib": 237.8,
|
||||
"collector_container_mib": 76.9,
|
||||
"queue": {
|
||||
"otlp_http/loki": 23.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 10.0
|
||||
},
|
||||
"alert": "pending"
|
||||
},
|
||||
{
|
||||
"t": 36.9,
|
||||
"collector_heap_mib": 53.4,
|
||||
"collector_rss_mib": 237.8,
|
||||
"collector_container_mib": 77.0,
|
||||
"queue": {
|
||||
"otlp_http/loki": 31.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 13.0
|
||||
},
|
||||
"alert": "pending"
|
||||
},
|
||||
{
|
||||
"t": 49.0,
|
||||
"collector_heap_mib": 62.1,
|
||||
"collector_rss_mib": 239.8,
|
||||
"collector_container_mib": 81.0,
|
||||
"queue": {
|
||||
"otlp_http/loki": 38.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 16.0
|
||||
},
|
||||
"alert": "pending"
|
||||
},
|
||||
{
|
||||
"t": 61.1,
|
||||
"collector_heap_mib": 62.1,
|
||||
"collector_rss_mib": 239.8,
|
||||
"collector_container_mib": 87.1,
|
||||
"queue": {
|
||||
"otlp_http/loki": 46.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 19.0
|
||||
},
|
||||
"alert": "pending"
|
||||
},
|
||||
{
|
||||
"t": 73.8,
|
||||
"collector_heap_mib": 45.9,
|
||||
"collector_rss_mib": 247.8,
|
||||
"collector_container_mib": 87.1,
|
||||
"queue": {
|
||||
"otlp_http/loki": 46.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 19.0
|
||||
},
|
||||
"alert": "pending"
|
||||
},
|
||||
{
|
||||
"t": 85.9,
|
||||
"collector_heap_mib": 54.5,
|
||||
"collector_rss_mib": 247.8,
|
||||
"collector_container_mib": 87.2,
|
||||
"queue": {
|
||||
"otlp_http/loki": 53.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 22.0
|
||||
},
|
||||
"alert": "firing"
|
||||
},
|
||||
{
|
||||
"t": 98.0,
|
||||
"collector_heap_mib": 66.1,
|
||||
"collector_rss_mib": 248.0,
|
||||
"collector_container_mib": 87.3,
|
||||
"queue": {
|
||||
"otlp_http/loki": 59.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 24.0
|
||||
},
|
||||
"alert": "firing"
|
||||
},
|
||||
{
|
||||
"t": 110.0,
|
||||
"collector_heap_mib": 75.7,
|
||||
"collector_rss_mib": 254.0,
|
||||
"collector_container_mib": 95.4,
|
||||
"queue": {
|
||||
"otlp_http/loki": 66.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 27.0
|
||||
},
|
||||
"alert": "firing"
|
||||
},
|
||||
{
|
||||
"t": 122.1,
|
||||
"collector_heap_mib": 84.7,
|
||||
"collector_rss_mib": 262.2,
|
||||
"collector_container_mib": 101.5,
|
||||
"queue": {
|
||||
"otlp_http/loki": 74.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 30.0
|
||||
},
|
||||
"alert": "firing"
|
||||
},
|
||||
{
|
||||
"t": 134.9,
|
||||
"collector_heap_mib": 84.7,
|
||||
"collector_rss_mib": 262.2,
|
||||
"collector_container_mib": 103.7,
|
||||
"queue": {
|
||||
"otlp_http/loki": 81.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 33.0
|
||||
},
|
||||
"alert": "firing"
|
||||
},
|
||||
{
|
||||
"t": 147.0,
|
||||
"collector_heap_mib": 64.8,
|
||||
"collector_rss_mib": 264.2,
|
||||
"collector_container_mib": 103.6,
|
||||
"queue": {
|
||||
"otlp_http/loki": 81.0,
|
||||
"otlp_http/prometheus": 0.0,
|
||||
"otlp_grpc/tempo": 33.0
|
||||
},
|
||||
"alert": "firing"
|
||||
}
|
||||
],
|
||||
"collector_heap_max_mib": 84.7,
|
||||
"collector_rss_max_mib": 264.2,
|
||||
"collector_container_max_mib": 103.7,
|
||||
"TelemetryExportFailing_firing_at_s": 85.9,
|
||||
"queue_drained_after_start_s": 55.3,
|
||||
"probe_traces": [
|
||||
{
|
||||
"sent_at_s": 0.0,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 12.1,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 24.8,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 36.9,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 49.0,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 61.1,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 73.8,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 85.9,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 98.0,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 110.0,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 122.1,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 134.9,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
},
|
||||
{
|
||||
"sent_at_s": 147.0,
|
||||
"http": 201,
|
||||
"trace_in_tempo": true
|
||||
}
|
||||
],
|
||||
"lost_in_collector": {
|
||||
"spans": 2048,
|
||||
"log_records": 213
|
||||
},
|
||||
"resolved_after_start_s": 70.6,
|
||||
"after": {
|
||||
"requests": 3489,
|
||||
"codes": {
|
||||
"201": 3489
|
||||
},
|
||||
"success_ratio": 1.0,
|
||||
"success_rps": 53.2,
|
||||
"p95_ms_successful": 32.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted
|
||||
# date: 2026-11-07 19:27:09, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/incidents.py telemetry-down
|
||||
# exit code: 0, 284.7 s
|
||||
|
||||
=== telemetry-down
|
||||
[ 22.2s] baseline: {'requests': 793, 'codes': {'201': 793}, 'success_ratio': 1.0, 'success_rps': 52.9, 'p95_ms_successful': 32.0}, collector heap 32.3 MiB, container 76.9 MiB
|
||||
[ 53.5s] docker compose stop tempo loki: stopped
|
||||
[ 55.6s] {'t': 0.0, 'collector_heap_mib': 51.5, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.9, 'queue': {'otlp_http/loki': 16.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 7.0}, 'alert': 'inactive'}
|
||||
[ 68.3s] {'t': 12.1, 'collector_heap_mib': 60.3, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.9, 'queue': {'otlp_http/loki': 16.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 7.0}, 'alert': 'inactive'}
|
||||
[ 80.4s] {'t': 24.8, 'collector_heap_mib': 44.3, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.9, 'queue': {'otlp_http/loki': 23.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'pending'}
|
||||
[ 92.4s] {'t': 36.9, 'collector_heap_mib': 53.4, 'collector_rss_mib': 237.8, 'collector_container_mib': 77.0, 'queue': {'otlp_http/loki': 31.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 13.0}, 'alert': 'pending'}
|
||||
[ 104.5s] {'t': 49.0, 'collector_heap_mib': 62.1, 'collector_rss_mib': 239.8, 'collector_container_mib': 81.0, 'queue': {'otlp_http/loki': 38.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 16.0}, 'alert': 'pending'}
|
||||
[ 117.3s] {'t': 61.1, 'collector_heap_mib': 62.1, 'collector_rss_mib': 239.8, 'collector_container_mib': 87.1, 'queue': {'otlp_http/loki': 46.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'}
|
||||
[ 129.4s] {'t': 73.8, 'collector_heap_mib': 45.9, 'collector_rss_mib': 247.8, 'collector_container_mib': 87.1, 'queue': {'otlp_http/loki': 46.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'}
|
||||
[ 141.4s] {'t': 85.9, 'collector_heap_mib': 54.5, 'collector_rss_mib': 247.8, 'collector_container_mib': 87.2, 'queue': {'otlp_http/loki': 53.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'firing'}
|
||||
[ 153.5s] {'t': 98.0, 'collector_heap_mib': 66.1, 'collector_rss_mib': 248.0, 'collector_container_mib': 87.3, 'queue': {'otlp_http/loki': 59.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 24.0}, 'alert': 'firing'}
|
||||
[ 165.6s] {'t': 110.0, 'collector_heap_mib': 75.7, 'collector_rss_mib': 254.0, 'collector_container_mib': 95.4, 'queue': {'otlp_http/loki': 66.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'}
|
||||
[ 178.4s] {'t': 122.1, 'collector_heap_mib': 84.7, 'collector_rss_mib': 262.2, 'collector_container_mib': 101.5, 'queue': {'otlp_http/loki': 74.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 30.0}, 'alert': 'firing'}
|
||||
[ 190.4s] {'t': 134.9, 'collector_heap_mib': 84.7, 'collector_rss_mib': 262.2, 'collector_container_mib': 103.7, 'queue': {'otlp_http/loki': 81.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 33.0}, 'alert': 'firing'}
|
||||
[ 202.5s] {'t': 147.0, 'collector_heap_mib': 64.8, 'collector_rss_mib': 264.2, 'collector_container_mib': 103.6, 'queue': {'otlp_http/loki': 81.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 33.0}, 'alert': 'firing'}
|
||||
[ 214.0s] docker compose start tempo loki
|
||||
[ 269.3s] exporter queues drained 55.3 s after start
|
||||
[ 284.6s] API during outage: {'requests': 8486, 'codes': {'201': 8486}, 'success_ratio': 1.0, 'success_rps': 52.9, 'p95_ms_successful': 32.0}
|
||||
[ 284.6s] probe traces: 13/13 reached Tempo (lost: sent at [] s)
|
||||
[ 284.6s] lost in Collector (send_failed+enqueue_failed): {'spans': 2048, 'log_records': 213}; collector heap 32.3 -> max 84.7 MiB (memory_limiter 200 MiB), container max 103.7 MiB
|
||||
[ 284.6s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-telemetry-down.json
|
||||
PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"scenario": "worker-stop",
|
||||
"date": "2026-11-07 19:15:03",
|
||||
"firing_after_stop_s": {
|
||||
"MessengerWorkerNoConsumers": 94.0,
|
||||
"MessengerQueueLagGrowing": 124.5
|
||||
},
|
||||
"diagnosis": {
|
||||
"queue_consumers": 0.0,
|
||||
"broker_queue_depth": 3.0,
|
||||
"jobs_backlog_queued": 3.0,
|
||||
"oldest_job_age_s": 115.4,
|
||||
"outbox_oldest_age_s": 0.0,
|
||||
"published_ok_last_5m": 12.39609660691289,
|
||||
"worker_metrics_last_seen_s": 123.0,
|
||||
"last_worker_log": {
|
||||
"event": "service.stopped",
|
||||
"message": "stopped"
|
||||
},
|
||||
"upload_trace_span_names": [
|
||||
"INSERT",
|
||||
"NOTIFY",
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"S3 PutObject",
|
||||
"SELECT",
|
||||
"UPDATE",
|
||||
"create attachments",
|
||||
"inspect image",
|
||||
"publish attachments",
|
||||
"redis GET"
|
||||
],
|
||||
"upload_trace_has_consumer_span": false
|
||||
},
|
||||
"ready_after_start_s": [
|
||||
4.9,
|
||||
5.0,
|
||||
5.4
|
||||
],
|
||||
"resolved_after_start_s": {
|
||||
"MessengerWorkerNoConsumers": 45.6,
|
||||
"MessengerQueueLagGrowing": 45.6
|
||||
},
|
||||
"after_recovery": {
|
||||
"jobs_backlog_queued": 0.0,
|
||||
"oldest_job_age_s": 0.0,
|
||||
"queue_consumers": 2.0
|
||||
},
|
||||
"time_to_ready_p95_5m_s": 27.8
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved
|
||||
# date: 2026-11-07 19:17:54, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/incidents.py worker-stop
|
||||
# exit code: 0, 170.7 s
|
||||
|
||||
=== worker-stop
|
||||
[ 0.2s] baseline: consumers=2.0, oldest job age=0.0 s
|
||||
[ 2.6s] docker compose stop worker
|
||||
[ 2.7s] uploaded 3 attachments: 202 queued (API accepts without worker)
|
||||
[ 2.8s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
|
||||
[ 7.9s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
|
||||
[ 13.0s] consumers=0.0 backlog queued=3.0 oldest age=5.1 s, alerts: {}
|
||||
[ 18.1s] consumers=0.0 backlog queued=3.0 oldest age=5.1 s, alerts: {}
|
||||
[ 23.1s] consumers=0.0 backlog queued=3.0 oldest age=15.1 s, alerts: {}
|
||||
[ 28.2s] consumers=0.0 backlog queued=3.0 oldest age=15.1 s, alerts: {}
|
||||
[ 33.3s] consumers=0.0 backlog queued=3.0 oldest age=25.2 s, alerts: {}
|
||||
[ 38.4s] consumers=0.0 backlog queued=3.0 oldest age=25.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 43.5s] consumers=0.0 backlog queued=3.0 oldest age=35.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 48.5s] consumers=0.0 backlog queued=3.0 oldest age=35.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 53.7s] consumers=0.0 backlog queued=3.0 oldest age=45.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 58.7s] consumers=0.0 backlog queued=3.0 oldest age=45.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 63.8s] consumers=0.0 backlog queued=3.0 oldest age=55.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 68.8s] consumers=0.0 backlog queued=3.0 oldest age=55.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 73.9s] consumers=0.0 backlog queued=3.0 oldest age=65.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 79.0s] consumers=0.0 backlog queued=3.0 oldest age=65.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 84.1s] consumers=0.0 backlog queued=3.0 oldest age=75.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 89.2s] consumers=0.0 backlog queued=3.0 oldest age=75.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||
[ 94.2s] ALERT MessengerWorkerNoConsumers firing (94.0 s after stop)
|
||||
[ 94.2s] consumers=0.0 backlog queued=3.0 oldest age=85.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||
[ 99.3s] consumers=0.0 backlog queued=3.0 oldest age=85.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||
[ 104.4s] consumers=0.0 backlog queued=3.0 oldest age=95.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||
[ 109.5s] consumers=0.0 backlog queued=3.0 oldest age=95.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||
[ 114.6s] consumers=0.0 backlog queued=3.0 oldest age=105.4 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||
[ 119.7s] consumers=0.0 backlog queued=3.0 oldest age=105.4 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||
[ 124.7s] ALERT MessengerQueueLagGrowing firing (124.5 s after stop)
|
||||
[ 124.8s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 3.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 115.4, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 12.39609660691289, "worker_metrics_last_seen_s": 123.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET"], "upload_trace_has_consumer_span": false}
|
||||
[ 128.9s] docker compose start worker
|
||||
[ 130.3s] all attachments ready 5.4 s after worker start; backlog now 3.0
|
||||
[ 130.3s] ... MessengerWorkerNoConsumers: firing (0 s)
|
||||
[ 135.3s] ... MessengerWorkerNoConsumers: firing (5 s)
|
||||
[ 140.3s] ... MessengerWorkerNoConsumers: firing (10 s)
|
||||
[ 145.4s] ... MessengerWorkerNoConsumers: firing (15 s)
|
||||
[ 150.4s] ... MessengerWorkerNoConsumers: firing (20 s)
|
||||
[ 155.4s] ... MessengerWorkerNoConsumers: firing (25 s)
|
||||
[ 160.4s] ... MessengerWorkerNoConsumers: firing (30 s)
|
||||
[ 165.5s] ... MessengerWorkerNoConsumers: firing (35 s)
|
||||
[ 170.5s] alerts resolved: {'MessengerWorkerNoConsumers': 45.6, 'MessengerQueueLagGrowing': 45.6}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0}
|
||||
[ 170.6s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-worker-stop.json
|
||||
PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,8 @@
|
||||
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
|
||||
# date: 2026-11-07 19:40:12, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py concurrency
|
||||
# exit code: 0, 19.2 s
|
||||
|
||||
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
|
||||
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.6 s after worker start, peak processing 2 of 8 (6 samples)
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
|
||||
# date: 2026-11-07 19:35:22, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
|
||||
# exit code: 0, 161.4 s
|
||||
|
||||
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
|
||||
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
|
||||
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
|
||||
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
|
||||
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
|
||||
staging source still present after the crash (worker died before cleanup); waiting 60 s window
|
||||
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
|
||||
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
|
||||
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 18 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,13 @@
|
||||
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
|
||||
# date: 2026-11-07 19:39:53, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_janitor_replay.py
|
||||
# exit code: 0, 192.2 s
|
||||
|
||||
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
waiting 84 s until the leftovers are older than grace (120 s)
|
||||
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
|
||||
ops replay -> job c532cc9d-8fd5-4baa-9dcf-cc7331f50086 ready
|
||||
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 2, deleted 2
|
||||
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,11 @@
|
||||
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
|
||||
# date: 2026-11-07 19:31:20, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py limits
|
||||
# exit code: 0, 74.8 s
|
||||
|
||||
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
|
||||
10 MiB + 8 bytes -> 413 before any job is created
|
||||
imaging.py on 8000x5000 PNG: exit 0, peak RSS 225.8 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
|
||||
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
|
||||
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,9 @@
|
||||
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
|
||||
# date: 2026-11-07 19:32:40, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py outbox-crash
|
||||
# exit code: 0, 20.3 s
|
||||
|
||||
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
|
||||
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 60ca1ef4-a5fb-48ad-94bd-5f14f683304d ready, attempt 1, no manual action
|
||||
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,16 @@
|
||||
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
|
||||
# date: 2026-11-07 19:32:20, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py worker-down broker-down
|
||||
# exit code: 0, 60.4 s
|
||||
|
||||
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
|
||||
worker stopped
|
||||
upload while worker is down -> 202 queued, attachment 5d7b65bc-2cb9-4b4a-801e-6c30173dd513
|
||||
message with attachment 201, content 409, other messages 201; job 6133fe0a-b568-4024-8bb7-4d7d32ed0433 published, queue messages/unacked = (1, 0)
|
||||
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
|
||||
worker started -> same job 6133fe0a-b568-4024-8bb7-4d7d32ed0433 ready, thumbnail downloadable; staging object deleted
|
||||
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
|
||||
rabbitmq stopped
|
||||
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 3e57742c-4cc4-48b0-9645-0e742fa323d0
|
||||
rabbitmq started -> dispatcher published the pending row -> ready
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
|
||||
# date: 2026-11-07 19:36:41, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq
|
||||
# exit code: 0, 78.5 s
|
||||
|
||||
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
|
||||
2 x storage_unavailable -> retries after 1.70 s and 2.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
|
||||
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
|
||||
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
|
||||
ops replay -> same job_id 3d109dba-1036-482a-a6fb-8116c7894b6d attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
|
||||
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
|
||||
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
|
||||
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
|
||||
schema-invalid message and unknown job_id -> reject -> DLQ (4 message(s)); dlq-replay kept them: ['unknown job', 'unknown job']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
|
||||
# date: 2026-11-07 19:30:05, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py variants
|
||||
# exit code: 0, 21.0 s
|
||||
|
||||
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
|
||||
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,26 @@
|
||||
# lab7-security: lab7: HTTPS only, headers, outsider ACL, logout, shared login rate limit, size limits, Argon2id, rehash, no credential leaks
|
||||
# date: 2026-11-07 19:08:02, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_lab7_security.py
|
||||
# exit code: 0, 17.3 s
|
||||
|
||||
port 8080: closed
|
||||
empty trust store: rejected by certificate verification (unable to get local issuer certificate)
|
||||
foreign CA: rejected by certificate verification (unable to get local issuer certificate)
|
||||
wrong host name: rejected by certificate verification (Hostname mismatch, certificate is not valid for 'evil.example'.)
|
||||
PASS: only HTTPS on 8443; untrusted CA and wrong host name fail certificate verification
|
||||
HSTS, nosniff, no Server header; X-Instance-Id is an opaque alias (c4ba1764f1a3)
|
||||
PASS: security headers
|
||||
outsider: 6 attempts on a foreign chat/attachment -> 404 (existence not revealed); X-User-Id of the victim is ignored
|
||||
PASS: ACL — outsider reads/writes nothing, cannot attach a foreign file
|
||||
PASS: logout -> the same token is 401 on every replica (session deleted in shared Redis)
|
||||
codes [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429, 429, 429]; answered by replicas ['76a44da5ae93', 'c4ba1764f1a3']; Retry-After 60 s
|
||||
PASS: login rate limit is shared by all replicas (Redis), 429 + Retry-After, before Argon2id
|
||||
JSON > 64 KiB -> 413 (with Content-Length and chunked); upload Content-Length 50 MB -> 413 at the LB
|
||||
PASS: request size limits
|
||||
stored: $argon2id$v=19$m=32768,t=2,p=1$<16-byte salt>$<hash>; two users with the same password have different hashes
|
||||
PASS: Argon2id with salt and parameters in the stored string
|
||||
legacy hash m=12288,t=3 -> after successful login m=32768,t=2,p=1; the new hash still verifies
|
||||
PASS: old password hash is upgraded on successful login
|
||||
errors do not echo credentials; Redis stores session:<SHA-256(token)> only; DB stores only the hash
|
||||
PASS: credentials and session tokens are not revealed
|
||||
9 passed, 0 failed
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"container_logs": {
|
||||
"password": 0,
|
||||
"token": 0,
|
||||
"basic_b64": 0
|
||||
},
|
||||
"container_logs_bearer_or_basic_header": 0,
|
||||
"loki": {
|
||||
"password": 0,
|
||||
"token": 0,
|
||||
"basic_b64": 0
|
||||
},
|
||||
"tempo": {
|
||||
"traces_fetched": 4,
|
||||
"password": 0,
|
||||
"token": 0,
|
||||
"basic_b64": 0,
|
||||
"authorization_attribute": 0
|
||||
},
|
||||
"prometheus_labels": {
|
||||
"token": 0,
|
||||
"user_id": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# leak-check: lab7: password/token/Authorization absent in container logs, Loki, Tempo traces, Prometheus labels
|
||||
# date: 2026-11-07 19:08:23, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/leak_check.py
|
||||
# exit code: 0, 21.5 s
|
||||
|
||||
{"container_logs": {"password": 0, "token": 0, "basic_b64": 0}, "container_logs_bearer_or_basic_header": 0, "loki": {"password": 0, "token": 0, "basic_b64": 0}, "tempo": {"traces_fetched": 4, "password": 0, "token": 0, "basic_b64": 0, "authorization_attribute": 0}, "prometheus_labels": {"token": 0, "user_id": 0}}
|
||||
searched: container logs, Loki, Tempo, Prometheus labels; traces fetched: 4
|
||||
PASS: no password, session token or Authorization value in logs or telemetry
|
||||
@@ -0,0 +1,749 @@
|
||||
[
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "INSERT outbox (enqueue)",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "UPDATE users.username",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "SELECT outbox",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table outbox"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "DELETE messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "DROP TABLE messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "must be owner of table messages"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "CREATE TABLE",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for schema public"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "ALTER ROLE (escalate)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied to alter role"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "PUT staging/",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "DELETE staging/ (rollback)",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "PUT attachments/ (worker's prefix)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "GET staging/ (upload sources)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "LIST bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "DELETE attachments/",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "CREATE bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password', 'redis_password', 's3_access_key_id', 's3_secret_access_key']"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/worker",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/dispatcher",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/janitor",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "GET session:*",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "GET other key",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "No permissions to access a key"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "KEYS *",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "User api has no permissions to run the 'keys' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "FLUSHALL",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "User api has no permissions to run the 'flushall' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "CONFIG GET",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "User api has no permissions to run the 'config|get' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "ACL LIST",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "User api has no permissions to run the 'acl|list' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "SUBSCRIBE / PUBLISH",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "User api has no permissions to run the 'publish' command"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "SELECT attachments",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users (passwords)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "SELECT messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "DELETE attachments",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table attachments"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "UPDATE outbox",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table outbox"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "PUT attachments/",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "LIST attachments/",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "LIST bucket root",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "PUT outside known prefixes",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "CREATE bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/api",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/dispatcher",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/janitor",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "SELECT outbox",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "SELECT messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "DELETE attachments",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table attachments"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password']"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/api",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/worker",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/janitor",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "SELECT attachments",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "INSERT messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "DELETE chats",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table chats"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "s3",
|
||||
"action": "LIST bucket",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "s3",
|
||||
"action": "GET object content",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "s3",
|
||||
"action": "CREATE bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/api",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/worker",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/dispatcher",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,115 @@
|
||||
# least-privilege: lab7: each service identity vs PostgreSQL grants, S3 prefixes, OpenBao policies, Redis ACL
|
||||
# date: 2026-11-07 19:07:44, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/test_least_privilege.py
|
||||
# exit code: 0, 5.5 s
|
||||
|
||||
|
||||
api -> postgres
|
||||
PASS SELECT users expect allowed observed allowed
|
||||
PASS INSERT outbox (enqueue) expect allowed observed allowed
|
||||
PASS UPDATE users.username expect denied observed denied
|
||||
PASS SELECT outbox expect denied observed denied
|
||||
PASS DELETE messages expect denied observed denied
|
||||
PASS DROP TABLE messages expect denied observed denied
|
||||
PASS CREATE TABLE expect denied observed denied
|
||||
PASS ALTER ROLE (escalate) expect denied observed denied
|
||||
|
||||
api -> s3
|
||||
PASS PUT staging/ expect allowed observed allowed
|
||||
PASS DELETE staging/ (rollback) expect allowed observed allowed
|
||||
PASS PUT attachments/ (worker's prefix) expect denied observed denied
|
||||
PASS GET staging/ (upload sources) expect denied observed denied
|
||||
PASS LIST bucket expect denied observed denied
|
||||
PASS DELETE attachments/ expect denied observed denied
|
||||
PASS CREATE bucket expect denied observed denied
|
||||
|
||||
api -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/worker expect denied observed denied
|
||||
PASS read messenger/dispatcher expect denied observed denied
|
||||
PASS read messenger/janitor expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
api -> redis
|
||||
PASS GET session:* expect allowed observed allowed
|
||||
PASS GET other key expect denied observed denied
|
||||
PASS KEYS * expect denied observed denied
|
||||
PASS FLUSHALL expect denied observed denied
|
||||
PASS CONFIG GET expect denied observed denied
|
||||
PASS ACL LIST expect denied observed denied
|
||||
PASS SUBSCRIBE / PUBLISH expect denied observed denied
|
||||
|
||||
worker -> postgres
|
||||
PASS SELECT attachments expect allowed observed allowed
|
||||
PASS SELECT users (passwords) expect denied observed denied
|
||||
PASS SELECT messages expect denied observed denied
|
||||
PASS DELETE attachments expect denied observed denied
|
||||
PASS UPDATE outbox expect denied observed denied
|
||||
|
||||
worker -> s3
|
||||
PASS PUT attachments/ expect allowed observed allowed
|
||||
PASS LIST attachments/ expect allowed observed allowed
|
||||
PASS LIST bucket root expect denied observed denied
|
||||
PASS PUT outside known prefixes expect denied observed denied
|
||||
PASS CREATE bucket expect denied observed denied
|
||||
|
||||
worker -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/api expect denied observed denied
|
||||
PASS read messenger/dispatcher expect denied observed denied
|
||||
PASS read messenger/janitor expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
dispatcher -> postgres
|
||||
PASS SELECT outbox expect allowed observed allowed
|
||||
PASS SELECT users expect denied observed denied
|
||||
PASS SELECT messages expect denied observed denied
|
||||
PASS DELETE attachments expect denied observed denied
|
||||
|
||||
dispatcher -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/api expect denied observed denied
|
||||
PASS read messenger/worker expect denied observed denied
|
||||
PASS read messenger/janitor expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
janitor -> postgres
|
||||
PASS SELECT attachments expect allowed observed allowed
|
||||
PASS SELECT users expect denied observed denied
|
||||
PASS INSERT messages expect denied observed denied
|
||||
PASS DELETE chats expect denied observed denied
|
||||
|
||||
janitor -> s3
|
||||
PASS LIST bucket expect allowed observed allowed
|
||||
PASS GET object content expect denied observed denied
|
||||
PASS CREATE bucket expect denied observed denied
|
||||
|
||||
janitor -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/api expect denied observed denied
|
||||
PASS read messenger/worker expect denied observed denied
|
||||
PASS read messenger/dispatcher expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
83 checks, 0 failed; saved to evidence/least-privilege.json
|
||||
PASS: every service identity can do its job and nothing else that was probed
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"old_serial": "70ea18ded7f86e5a",
|
||||
"new_serial": "22a71730433d9d42",
|
||||
"probe": {
|
||||
"valid LB identity (CN=lb)": "http_200",
|
||||
"foreign CA, CN=lb": "connection_closed_after_handshake",
|
||||
"our CA, expired, CN=lb": "connection_closed_after_handshake",
|
||||
"previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"
|
||||
},
|
||||
"api_events_after_revoke": {
|
||||
"tls.crl_loaded": 2,
|
||||
"tls.revoked_connections_closed": 2,
|
||||
"tls.client_rejected": 4
|
||||
},
|
||||
"load": {
|
||||
"seconds": 23.1,
|
||||
"codes": {
|
||||
"me:200": 243,
|
||||
"send:201": 243,
|
||||
"login:201": 11
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-rotate-leaf: lab7: LB leaf rotated (HAProxy reload), old leaf revoked via CRL, rejected by API
|
||||
# date: 2026-11-07 19:09:33, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/lifecycle.py rotate-leaf
|
||||
# exit code: 0, 27.6 s
|
||||
|
||||
--- rotate-leaf: клиентский сертификат LB (CN=lb)
|
||||
{"old_serial": "70ea18ded7f86e5a", "new_serial": "22a71730433d9d42", "probe": {"valid LB identity (CN=lb)": "http_200", "foreign CA, CN=lb": "connection_closed_after_handshake", "our CA, expired, CN=lb": "connection_closed_after_handshake", "previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"}, "api_events_after_revoke": {"tls.crl_loaded": 2, "tls.revoked_connections_closed": 2, "tls.client_rejected": 4}, "load": {"seconds": 23.1, "codes": {"me:200": 243, "send:201": 243, "login:201": 11}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors
|
||||
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-rotate-leaf.json
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"kv_version": [
|
||||
5,
|
||||
6
|
||||
],
|
||||
"replicas": 2,
|
||||
"active_passwords_during_overlap": 2,
|
||||
"all_replicas_picked_up_after_s": 23.9,
|
||||
"active_password_hashes_after": 1,
|
||||
"new_password_hash_matches": true,
|
||||
"load": {
|
||||
"seconds": 37.6,
|
||||
"codes": {
|
||||
"me:200": 431,
|
||||
"send:201": 431,
|
||||
"login:201": 18
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-rotate-secret: lab7: Redis password of api rotated via OpenBao under load, old one revoked
|
||||
# date: 2026-11-07 19:09:06, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/lifecycle.py rotate-secret
|
||||
# exit code: 0, 38.2 s
|
||||
|
||||
--- rotate-secret: пароль Redis ACL-пользователя api
|
||||
{"kv_version": [5, 6], "replicas": 2, "active_passwords_during_overlap": 2, "all_replicas_picked_up_after_s": 23.9, "active_password_hashes_after": 1, "new_password_hash_matches": true, "load": {"seconds": 37.6, "codes": {"me:200": 431, "send:201": 431, "login:201": 18}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors
|
||||
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-rotate-secret.json
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"sealed_seconds": 75.6,
|
||||
"restarted_replica_health_while_sealed": "starting",
|
||||
"replicas_answering_while_sealed": 1,
|
||||
"restarted_replica_ready_after_unseal_s": 5.4,
|
||||
"api_events": {
|
||||
"secrets.refresh_failed": 5,
|
||||
"secrets.waiting": 8,
|
||||
"secrets.login": 1
|
||||
},
|
||||
"load": {
|
||||
"seconds": 89.2,
|
||||
"codes": {
|
||||
"me:200": 1012,
|
||||
"send:201": 1012,
|
||||
"login:201": 43
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-sealed: lab7: sealed OpenBao: running replicas serve, restarted replica not ready until unseal
|
||||
# date: 2026-11-07 19:12:47, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/lifecycle.py sealed
|
||||
# exit code: 0, 90.8 s
|
||||
|
||||
--- sealed: OpenBao запечатан
|
||||
{"sealed_seconds": 75.6, "restarted_replica_health_while_sealed": "starting", "replicas_answering_while_sealed": 1, "restarted_replica_ready_after_unseal_s": 5.4, "api_events": {"secrets.refresh_failed": 5, "secrets.waiting": 8, "secrets.login": 1}, "load": {"seconds": 89.2, "codes": {"me:200": 1012, "send:201": 1012, "login:201": 43}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready (fail closed) until unseal
|
||||
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-sealed.json
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"role_ttl": "20s / max 40s",
|
||||
"observed_seconds": 100,
|
||||
"api_events": {
|
||||
"secrets.refresh_failed": 2,
|
||||
"secrets.login": 4
|
||||
},
|
||||
"load": {
|
||||
"seconds": 101.6,
|
||||
"codes": {
|
||||
"me:200": 1172,
|
||||
"send:201": 1172,
|
||||
"login:201": 49
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-token-expiry: lab7: OpenBao token TTL 20 s / max 40 s + revoke: services re-login with their cert
|
||||
# date: 2026-11-07 19:11:16, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/lifecycle.py token-expiry
|
||||
# exit code: 0, 102.4 s
|
||||
|
||||
--- token-expiry: короткий TTL токенов OpenBao роли api
|
||||
{"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": {"secrets.refresh_failed": 2, "secrets.login": 4}, "load": {"seconds": 101.6, "codes": {"me:200": 1172, "send:201": 1172, "login:201": 49}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected
|
||||
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-token-expiry.json
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 7,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.016,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 6202,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 206.623,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.0,
|
||||
"p95": 32.0,
|
||||
"p99": 47.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# load: public load generator, 30 s, concurrency 4
|
||||
# date: 2026-11-07 19:45:30, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe tests/load.py --base-url https://127.0.0.1:8443 --ca-file C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\secrets\pki\public\ca.crt --duration 30 --concurrency 4 --output C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\load.json
|
||||
# exit code: 0, 30.5 s
|
||||
|
||||
{
|
||||
"lab": 7,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.016,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 6202,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 206.623,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.0,
|
||||
"p95": 32.0,
|
||||
"p99": 47.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"baseline": {
|
||||
"legit": {
|
||||
"requests": 694,
|
||||
"codes": {
|
||||
"200": 347,
|
||||
"201": 347
|
||||
},
|
||||
"p50_ms": 12.3,
|
||||
"p95_ms": 34.6,
|
||||
"p99_ms": 35.5
|
||||
},
|
||||
"attack": null
|
||||
},
|
||||
"under_attack": {
|
||||
"legit": {
|
||||
"requests": 186,
|
||||
"codes": {
|
||||
"200": 93,
|
||||
"201": 93
|
||||
},
|
||||
"p50_ms": 91.1,
|
||||
"p95_ms": 194.1,
|
||||
"p99_ms": 243.6
|
||||
},
|
||||
"attack": {
|
||||
"requests": 1924,
|
||||
"codes": {
|
||||
"401": 360,
|
||||
"429": 1564
|
||||
},
|
||||
"rps": 96.2,
|
||||
"p50_ms": 116.8,
|
||||
"p95_ms": 480.5
|
||||
}
|
||||
},
|
||||
"seconds": 20,
|
||||
"attackers": 16,
|
||||
"burst_one_replica": {
|
||||
"requests": 200,
|
||||
"codes": {
|
||||
"hashed": 75,
|
||||
"rejected_503": 125
|
||||
},
|
||||
"max_parallel_hashes": 2,
|
||||
"limit": 2,
|
||||
"queue_s": 2,
|
||||
"seconds": 2.11,
|
||||
"slowest_rejection_s": 2.05,
|
||||
"maxrss_growth_mib": 74.7
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# login-dos: lab7: wrong-password login flood: Argon2id budget per replica, 503/429, latency of legit requests
|
||||
# date: 2026-11-07 19:48:14, commit: 20afc2c + uncommitted changes
|
||||
# command: python.exe student-tests/login_dos.py
|
||||
# exit code: 0, 47.5 s
|
||||
|
||||
baseline 20 s ...
|
||||
{"legit": {"requests": 694, "codes": {"200": 347, "201": 347}, "p50_ms": 12.3, "p95_ms": 34.6, "p99_ms": 35.5}, "attack": null}
|
||||
attack: 16 threads of wrong-password logins over 40 usernames ...
|
||||
{"legit": {"requests": 186, "codes": {"200": 93, "201": 93}, "p50_ms": 91.1, "p95_ms": 194.1, "p99_ms": 243.6}, "attack": {"requests": 1924, "codes": {"401": 360, "429": 1564}, "rps": 96.2, "p50_ms": 116.8, "p95_ms": 480.5}}
|
||||
burst of 200 simultaneous password checks in one api process: {"requests": 200, "codes": {"hashed": 75, "rejected_503": 125}, "max_parallel_hashes": 2, "limit": 2, "queue_s": 2, "seconds": 2.11, "slowest_rejection_s": 2.05, "maxrss_growth_mib": 74.7}
|
||||
attack: 96.2 login attempts/s; 401 (hash computed) 360, 503 busy (rejected before hashing) 0, 429 rate limited 1564
|
||||
legit p95: 34.6 ms -> 194.1 ms; legit errors: {}
|
||||
saved evidence/login-dos.json
|
||||
@@ -0,0 +1,29 @@
|
||||
# Прогон в нативной песочнице (без Docker)
|
||||
|
||||
Решение собиралось в среде, где Docker Hub недоступен. Чтобы проверить код и конфигурации до Docker-прогона,
|
||||
тот же стенд запускался процессами на одной машине (Linux x86_64, 2 vCPU, 8 GiB на всё сразу): каждый сервис —
|
||||
на своём loopback-адресе с именем из compose (`/etc/hosts`: postgres, redis, s3, rabbitmq, openbao, otel-collector,
|
||||
prometheus), конфиги — файлы из `infra/` с подстановкой путей `/run/tls*` → `secrets/pki/...`; команды `docker` /
|
||||
`docker compose`, которые вызывают тесты и hooks, выполнялись подменой, управляющей процессами (stop/start/
|
||||
restart/kill/pause, exec в окружении сервиса, политика `restart: unless-stopped`).
|
||||
|
||||
Стенд поднят с нуля шагами оператора из REPORT.md: `pki.py init` → `init_secrets.py` → OpenBao → `bao.py init`,
|
||||
`unseal`, `configure` → остальные сервисы. Затем `python student-tests/collect_evidence.py <шаги>` с
|
||||
`EVIDENCE_DIR=evidence/native-sandbox` — результаты в этой папке, сводка — `summary.md`.
|
||||
|
||||
| Компонент | В песочнице | В compose.yaml |
|
||||
| --- | --- | --- |
|
||||
| Приложение (api ×2, worker, dispatcher, janitor, migrate, s3-init) | тот же код, Python 3.11, `requirements.txt` | образ из `Dockerfile` (python:3.11.13-slim) |
|
||||
| PostgreSQL | 16.15 (TLS, `pg_hba.conf` из infra) | 17.11-alpine |
|
||||
| Redis | 7.0.15 (`redis.conf` из infra, TLS + ACL) | 7.4.11-alpine |
|
||||
| SeaweedFS | 4.52 (те же флаги и `security.toml`) | 4.52 |
|
||||
| RabbitMQ | 4.4.0 (`rabbitmq.conf` из infra, EXTERNAL) | 4.1.4-alpine |
|
||||
| OpenBao | **HashiCorp Vault 2.1.2** — API-совместимая замена (sys/init, unseal, cert auth, KV v2, policies — те же пути и поля) | openbao/openbao:2.6.2 |
|
||||
| HAProxy (lb, otel-gate) | 2.8.30 (конфиги из infra; вместо DNS Docker — две статические реплики) | 3.2.25-alpine |
|
||||
| OTel Collector | заглушка OTLP/HTTP на 127.0.0.1:4319 за настоящим otel-gate (сохраняет пакеты для `leak_check.py`) | otelcol-contrib 0.164.0 |
|
||||
| Prometheus | 2.45 — только `web.yml` (TLS, `client_allowed_sans`) | v3.15.1 |
|
||||
| Loki, Tempo, Grafana | не запускались (нет бинарников) — их TLS проверяется только в Docker | 3.7.9, 3.1.1, 12.4.14 |
|
||||
|
||||
Не выполнялись в песочнице (нужен Docker или Loki/Tempo/Grafana): `stack`, `hardening --runtime` (статическая
|
||||
часть — да), `configs`, `telemetry*`, `dashboard`, `incident-*`, `upload-memory`, `s3-security`
|
||||
(`test_no_open_ports_or_console`), `rolling-update`, `failover-metrics`, `test_expiry.py`.
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"rows": [
|
||||
{
|
||||
"m_kib": 19456,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 23.4,
|
||||
"verify_ms": 23.6,
|
||||
"max_logins_per_s_per_replica": 84.7
|
||||
},
|
||||
{
|
||||
"m_kib": 19456,
|
||||
"t": 3,
|
||||
"p": 1,
|
||||
"hash_ms": 42.2,
|
||||
"verify_ms": 42.1,
|
||||
"max_logins_per_s_per_replica": 47.5
|
||||
},
|
||||
{
|
||||
"m_kib": 32768,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 51.2,
|
||||
"verify_ms": 49.9,
|
||||
"max_logins_per_s_per_replica": 40.1
|
||||
},
|
||||
{
|
||||
"m_kib": 47104,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 63.9,
|
||||
"verify_ms": 62.1,
|
||||
"max_logins_per_s_per_replica": 32.2
|
||||
},
|
||||
{
|
||||
"m_kib": 65536,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"hash_ms": 115.8,
|
||||
"verify_ms": 115.8,
|
||||
"max_logins_per_s_per_replica": 17.3
|
||||
},
|
||||
{
|
||||
"m_kib": 65536,
|
||||
"t": 3,
|
||||
"p": 1,
|
||||
"hash_ms": 195.8,
|
||||
"verify_ms": 187.2,
|
||||
"max_logins_per_s_per_replica": 10.7
|
||||
}
|
||||
],
|
||||
"maxrss_mib": 136.8,
|
||||
"current": [
|
||||
32768,
|
||||
2,
|
||||
1
|
||||
],
|
||||
"max_concurrency": 2,
|
||||
"queue_s": 2
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
# argon2-bench: lab7: Argon2id cost inside the api container for candidate parameters
|
||||
# date: 2026-11-07 15:08:10, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/argon2_bench.py
|
||||
# exit code: 0, 7.5 s
|
||||
|
||||
m (KiB) t p hash ms verify ms logins/s per replica at concurrency 2 RAM for hashing
|
||||
19456 2 1 23.4 23.6 84.7 38 MiB
|
||||
19456 3 1 42.2 42.1 47.5 38 MiB
|
||||
32768 2 1 51.2 49.9 40.1 64 MiB <- current
|
||||
47104 2 1 63.9 62.1 32.2 92 MiB
|
||||
65536 2 1 115.8 115.8 17.3 128 MiB
|
||||
65536 3 1 195.8 187.2 10.7 128 MiB
|
||||
process max RSS during the bench: 136.8 MiB
|
||||
saved evidence/argon2-bench.json
|
||||
@@ -0,0 +1,96 @@
|
||||
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||
# date: 2026-11-07 15:20:02, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_attachments.py
|
||||
# exit code: 0, 14.3 s
|
||||
|
||||
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||
exactly 10485760 bytes (10 MiB) -> 202, then ready
|
||||
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||
PASS: the 10 MiB limit is exact
|
||||
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||
PASS: oversized streamed body is cut off, the API keeps working
|
||||
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 889591f7
|
||||
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||
PASS: pixel limit is checked before decompression
|
||||
PNG cut in half (55908 bytes, image/png) -> 415
|
||||
PNG without IEND (111805 bytes, image/png) -> 415
|
||||
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||
PNG signature only (8 bytes, image/png) -> 415
|
||||
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
|
||||
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||
random bytes (1600 bytes, image/png) -> 415
|
||||
plain text (28 bytes, image/jpeg) -> 415
|
||||
empty body -> 400
|
||||
PASS: damaged and foreign files are rejected with 415, never 500
|
||||
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
|
||||
IEND with data (70 bytes, valid CRCs) -> 415
|
||||
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||
PNG bytes sent as image/jpeg -> 415
|
||||
JPEG bytes sent as image/png -> 415
|
||||
GIF bytes sent as image/png -> 415
|
||||
PNG sent as image/gif -> 415
|
||||
PNG sent as text/html -> 415
|
||||
PNG sent as application/octet-stream -> 415
|
||||
PNG sent as multipart/form-data -> 415
|
||||
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||
no filename -> 400
|
||||
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||
PASS: filename never becomes a path; dangerous names are refused
|
||||
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
|
||||
PASS: only chat members can see an attachment
|
||||
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||
attachment of another chat (author is a member of both) -> 404
|
||||
own + foreign attachment -> 404
|
||||
unknown attachment_id -> 404
|
||||
outsider referencing an attachment of the chat -> 404
|
||||
duplicate ids -> 400
|
||||
11 ids -> 400
|
||||
bad UUID -> 400
|
||||
empty text without attachments -> 400
|
||||
whitespace text without attachments -> 400
|
||||
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||
PASS: Idempotency-Key covers attachment_ids
|
||||
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||
PASS: an interrupted upload is dropped
|
||||
12 passed, 0 failed
|
||||
@@ -0,0 +1,18 @@
|
||||
# balancing: balancing, shared sessions/data/files, X-Instance-Id
|
||||
# date: 2026-11-07 15:21:37, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_balancing.py
|
||||
# exit code: 0, 2.8 s
|
||||
|
||||
20 x GET /users/me with one token -> replicas: {'4ec098d7198a': 10, '9b3c6b6625a9': 10}
|
||||
PASS: one session works on every replica, no second login (sessions are shared in Redis)
|
||||
message written by 9b3c6b6625a9; read back by ['4ec098d7198a', '9b3c6b6625a9']
|
||||
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
|
||||
logout on 4ec098d7198a; old token then rejected by ['4ec098d7198a', '9b3c6b6625a9']
|
||||
PASS: logout on one replica revokes the session on all of them
|
||||
first POST on 4ec098d7198a -> 201 34d01e03-4930-4859-b912-60d58ae2bca9; same key on 9b3c6b6625a9 -> 201 34d01e03-4930-4859-b912-60d58ae2bca9
|
||||
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
|
||||
file uploaded via 4ec098d7198a; downloaded byte for byte via ['4ec098d7198a', '9b3c6b6625a9']
|
||||
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
|
||||
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running replica ['4ec098d7198a', '9b3c6b6625a9']; api containers -> aliases: {'api-1': '4ec098d7198a', 'api-2': '9b3c6b6625a9'}
|
||||
PASS: X-Instance-Id is in every response and points to a real container
|
||||
6 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# check: offline contract check (make check)
|
||||
# date: 2026-11-07 15:07:32, commit: 5bfc53a + uncommitted changes
|
||||
# command: python scripts/check_contract.py
|
||||
# exit code: 0, 0.1 s
|
||||
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,47 @@
|
||||
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
|
||||
# date: 2026-11-07 15:34:26, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_consistency.py
|
||||
# exit code: 0, 95.1 s
|
||||
|
||||
declared 9 000 000 bytes, sent 211817, closed the connection
|
||||
declared 9 000 000 bytes, sent 3111817, closed the connection
|
||||
rows in attachments: 69 -> 69; objects in bucket: 264 -> 264
|
||||
temporary upload files in api containers: {'api-1': 0, 'api-2': 0} (removed when the request ends)
|
||||
api containers have no volumes: the original is never stored locally
|
||||
PASS: an interrupted upload leaves no row, no object and no local file
|
||||
crash after S3 PUT: row bcd86485 state=uploading, object staging/dcc998514ccb4f5385947581f4c192cc exists in S3
|
||||
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
|
||||
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 269, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.192} -> fresh row and object are kept (the upload might still be running)
|
||||
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 269, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.207} -> nothing deleted
|
||||
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.181} -> row and object deleted
|
||||
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.185} -> idempotent
|
||||
the ready attachment of the same chat is untouched
|
||||
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
|
||||
object staging/f07a7443c23b46d3ad398824008ba3b6 put into the bucket without a row in PostgreSQL
|
||||
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 273, "orphan_objects": 5, "orphan_objects_deleted": 5, "missing_objects": 1, "seconds": 0.208} -> fresh object kept
|
||||
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.203} -> reported, kept
|
||||
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.198} -> orphan deleted
|
||||
ready attachments older than the grace period are untouched (they have rows)
|
||||
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500)
|
||||
PASS: orphans are found and deleted only after the grace period; missing objects are reported
|
||||
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
|
||||
upload -> 503 unavailable in 386 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose start s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 202 (16.2 s after the old file became readable)
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 1, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 278, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.197}
|
||||
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
--- S3 frozen (no answer, timeouts): docker compose pause s3
|
||||
upload -> 503 unavailable in 4064 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose unpause s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 202 (0.3 s after the old file became readable)
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 287, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.195}
|
||||
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
5 passed, 0 failed
|
||||
@@ -0,0 +1,21 @@
|
||||
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
|
||||
# date: 2026-11-07 15:21:56, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_db.py
|
||||
# exit code: 0, 7.0 s
|
||||
|
||||
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 62 -> 63
|
||||
PASS: chat row was inserted and then rolled back together with members
|
||||
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
|
||||
PASS: the database itself rejects a message from a non-member
|
||||
stored password: $argon2id$v=19$m=32768,t=2,p=1$... (length 97)
|
||||
PASS: password is stored as Argon2id hash
|
||||
unique constraint on users: UNIQUE (username)
|
||||
PASS: username is unique in the database
|
||||
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; running the migrate service once more
|
||||
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; log: Migrations applied: none (schema is up to date)
|
||||
PASS: migrations are applied once, a repeated run does nothing
|
||||
api replicas that checked the schema at start: 5; api log lines 'Migrations applied': 0; migrate service: Exited (0)
|
||||
PASS: only the migrate service applies migrations, replicas wait for the schema
|
||||
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
|
||||
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
|
||||
7 passed, 0 failed
|
||||
@@ -0,0 +1,99 @@
|
||||
=== Data volume
|
||||
messages_total | messages_in_big_chat
|
||||
----------------+----------------------
|
||||
23017 | 15000
|
||||
|
||||
|
||||
=== History index
|
||||
indexdef
|
||||
----------------------------------------------------------------------------------------
|
||||
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
|
||||
|
||||
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.013..0.125 rows=51 loops=1)
|
||||
Buffers: shared hit=55
|
||||
-> Index Scan using messages_history on messages (actual time=0.012..0.119 rows=51 loops=1)
|
||||
Index Cond: (chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid)
|
||||
Buffers: shared hit=55
|
||||
Planning:
|
||||
Buffers: shared hit=6
|
||||
Planning Time: 0.119 ms
|
||||
Execution Time: 0.141 ms
|
||||
|
||||
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.011..0.030 rows=51 loops=1)
|
||||
Buffers: shared hit=6
|
||||
-> Index Scan using messages_history on messages (actual time=0.010..0.025 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 05:46:40+03'::timestamp with time zone, '1e03049f-75da-40e8-9821-2d439e107e51'::uuid)))
|
||||
Buffers: shared hit=6
|
||||
Planning Time: 0.124 ms
|
||||
Execution Time: 0.058 ms
|
||||
|
||||
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.036..0.142 rows=51 loops=1)
|
||||
Buffers: shared hit=60
|
||||
-> Index Scan using messages_history on messages m (actual time=0.035..0.137 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 05:46:40+03'::timestamp with time zone, '1e03049f-75da-40e8-9821-2d439e107e51'::uuid)))
|
||||
Buffers: shared hit=60
|
||||
SubPlan 1
|
||||
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
|
||||
Sort Key: a."position"
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.001..0.001 rows=0 loops=51)
|
||||
Recheck Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Index Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
Planning:
|
||||
Buffers: shared hit=63
|
||||
Planning Time: 0.255 ms
|
||||
Execution Time: 0.202 ms
|
||||
|
||||
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
QUERY PLAN
|
||||
---------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=2.823..2.840 rows=51 loops=1)
|
||||
Buffers: shared hit=311
|
||||
-> Index Scan using messages_history on messages (actual time=0.020..2.302 rows=10051 loops=1)
|
||||
Index Cond: (chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid)
|
||||
Buffers: shared hit=311
|
||||
Planning Time: 0.082 ms
|
||||
Execution Time: 2.859 ms
|
||||
|
||||
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
QUERY PLAN
|
||||
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=3.356..3.366 rows=51 loops=1)
|
||||
Buffers: shared hit=318
|
||||
-> Sort (actual time=3.354..3.359 rows=51 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 31kB
|
||||
Buffers: shared hit=318
|
||||
-> Seq Scan on messages (actual time=1.132..2.326 rows=5000 loops=1)
|
||||
Filter: ((chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 05:46:40+03'::timestamp with time zone, '1e03049f-75da-40e8-9821-2d439e107e51'::uuid)))
|
||||
Rows Removed by Filter: 18017
|
||||
Buffers: shared hit=318
|
||||
Planning:
|
||||
Buffers: shared hit=4
|
||||
Planning Time: 0.103 ms
|
||||
Execution Time: 3.385 ms
|
||||
|
||||
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
explain_users_left
|
||||
--------------------
|
||||
0
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
|
||||
# date: 2026-11-07 15:21:57, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/explain.py
|
||||
# exit code: 0, 0.9 s
|
||||
|
||||
=== Data volume
|
||||
=== History index
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
Buffers: shared hit=55
|
||||
-> Index Scan using messages_history on messages (actual time=0.012..0.119 rows=51 loops=1)
|
||||
Buffers: shared hit=55
|
||||
Buffers: shared hit=6
|
||||
Execution Time: 0.141 ms
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
Buffers: shared hit=6
|
||||
-> Index Scan using messages_history on messages (actual time=0.010..0.025 rows=51 loops=1)
|
||||
Buffers: shared hit=6
|
||||
Execution Time: 0.058 ms
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
Buffers: shared hit=60
|
||||
-> Index Scan using messages_history on messages m (actual time=0.035..0.137 rows=51 loops=1)
|
||||
Buffers: shared hit=60
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.001..0.001 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
Buffers: shared hit=63
|
||||
Execution Time: 0.202 ms
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
Buffers: shared hit=311
|
||||
-> Index Scan using messages_history on messages (actual time=0.020..2.302 rows=10051 loops=1)
|
||||
Buffers: shared hit=311
|
||||
Execution Time: 2.859 ms
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
Buffers: shared hit=318
|
||||
Buffers: shared hit=318
|
||||
-> Seq Scan on messages (actual time=1.132..2.326 rows=5000 loops=1)
|
||||
Buffers: shared hit=318
|
||||
Buffers: shared hit=4
|
||||
Execution Time: 3.385 ms
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
PASS: full plans saved to evidence/explain-history.txt
|
||||
@@ -0,0 +1,29 @@
|
||||
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
|
||||
# date: 2026-11-07 15:59:02, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/run_failover.py
|
||||
# exit code: 0, 40.8 s
|
||||
|
||||
hook: scripts/stop-one.sh; api replicas: ['api-1', 'api-2']
|
||||
=== run 1: api containers before
|
||||
api-1 api-1 Up (healthy)
|
||||
api-2 api-2 Up (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.82}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
api-1 api-1 Up (healthy)
|
||||
api-2 api-2 Exited (0)
|
||||
the hook stopped api-2 (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start api-2 -> healthy and back in HAProxy
|
||||
=== run 2: api containers before
|
||||
api-1 api-1 Up (healthy)
|
||||
api-2 api-2 Up (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.721}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
api-1 api-1 Exited (0)
|
||||
api-2 api-2 Up (healthy)
|
||||
the hook stopped api-1 (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start api-1 -> healthy and back in HAProxy
|
||||
PASS: public failover test passed 2 times; every api replica was stopped once and restored
|
||||
@@ -0,0 +1,29 @@
|
||||
# failures: Redis and PostgreSQL outages via the balancer
|
||||
# date: 2026-11-07 15:24:59, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_failures.py
|
||||
# exit code: 0, 33.8 s
|
||||
|
||||
stopping redis
|
||||
/health/live -> 200, /health/ready -> 503 (8 ms)
|
||||
valid token: GET /users/me -> 503 unavailable (7 ms), session cannot be checked
|
||||
unknown token -> 503, not 200: nobody is trusted without a check
|
||||
no token -> 401
|
||||
login -> 503: there is nowhere to store the session
|
||||
registration -> 201: it needs only PostgreSQL
|
||||
starting redis
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: Redis outage gives 503 and does not let anyone in without a session check
|
||||
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
|
||||
right after the crash: GET /users/me -> 503
|
||||
starting redis
|
||||
after start: the session issued before the crash still works (AOF, appendfsync always)
|
||||
PASS: sessions survive a Redis crash, not only a clean restart
|
||||
stopping postgres
|
||||
/health/live -> 200, /health/ready -> 503 (1007 ms)
|
||||
valid token: users/me, chats, history, new message -> 503 unavailable, no details
|
||||
no token -> 401, unknown token -> 401: anonymous users are not let in
|
||||
login -> 503, registration -> 503
|
||||
starting postgres
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
|
||||
3 passed, 0 failed
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
# idempotency: Idempotency-Key, parallel same key via replicas
|
||||
# date: 2026-11-07 15:21:41, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_idempotency.py
|
||||
# exit code: 0, 3.8 s
|
||||
|
||||
first -> 201; same JSON written differently -> 201; plain retry -> 201
|
||||
PASS: a retry with the same key returns the original 201 and message, no duplicate
|
||||
20 parallel POSTs with one key: statuses {201: 20}, replicas {'9b3c6b6625a9': 10, '4ec098d7198a': 10}, different bodies in answers: 1, messages in history: 1
|
||||
PASS: parallel requests with one key through different replicas create exactly one message
|
||||
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
|
||||
PASS: the first body wins, the same body gets 201, the other body gets 409
|
||||
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
|
||||
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
|
||||
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
|
||||
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
|
||||
key used by a 400 and by a 404 first, then valid request -> 201
|
||||
PASS: rejected requests (400, 404) do not store a result for the key
|
||||
same key and body by an outsider -> 404; by alice after logout -> 401
|
||||
PASS: a retry checks the session and chat access again, the saved answer is not leaked
|
||||
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
|
||||
PASS: an unconfirmed POST can be safely retried with the same key
|
||||
8 passed, 0 failed
|
||||
@@ -0,0 +1,8 @@
|
||||
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
|
||||
# date: 2026-11-07 15:19:47, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py concurrency
|
||||
# exit code: 0, 6.2 s
|
||||
|
||||
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
|
||||
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 3.0 s after worker start, peak processing 2 of 8 (17 samples)
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
|
||||
# date: 2026-11-07 15:16:04, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
|
||||
# exit code: 0, 133.0 s
|
||||
|
||||
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
|
||||
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
|
||||
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
|
||||
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
|
||||
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
|
||||
staging source still present after the crash (worker died before cleanup); waiting 60 s window
|
||||
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
|
||||
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
|
||||
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,13 @@
|
||||
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
|
||||
# date: 2026-11-07 15:19:41, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_janitor_replay.py
|
||||
# exit code: 0, 172.4 s
|
||||
|
||||
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
waiting 89 s until the leftovers are older than grace (120 s)
|
||||
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
|
||||
ops replay -> job bdf4dc61-b38c-4cf8-9a5a-0a5bbf7a7947 ready
|
||||
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 1, deleted 1
|
||||
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,11 @@
|
||||
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
|
||||
# date: 2026-11-07 15:13:13, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py limits
|
||||
# exit code: 0, 66.6 s
|
||||
|
||||
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
|
||||
10 MiB + 8 bytes -> 413 before any job is created
|
||||
imaging.py on 8000x5000 PNG: exit 0, peak RSS 223.8 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
|
||||
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
|
||||
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,9 @@
|
||||
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
|
||||
# date: 2026-11-07 15:13:51, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py outbox-crash
|
||||
# exit code: 0, 6.5 s
|
||||
|
||||
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
|
||||
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job a39e310d-6e0e-425c-baa6-33db9da3e290 ready, attempt 1, no manual action
|
||||
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,16 @@
|
||||
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
|
||||
# date: 2026-11-07 15:13:44, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py worker-down broker-down
|
||||
# exit code: 0, 30.6 s
|
||||
|
||||
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
|
||||
worker stopped
|
||||
upload while worker is down -> 202 queued, attachment 6f25aa5c-c235-40f8-bf9f-520404807cab
|
||||
message with attachment 201, content 409, other messages 201; job 182a3066-efb5-453d-8d2a-c906da4ac027 published, queue messages/unacked = (1, 1)
|
||||
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
|
||||
worker started -> same job 182a3066-efb5-453d-8d2a-c906da4ac027 ready, thumbnail downloadable; staging object deleted
|
||||
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
|
||||
rabbitmq stopped
|
||||
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 4d076cc6-0e4e-4ec7-ba03-10b557817907
|
||||
rabbitmq started -> dispatcher published the pending row -> ready
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
|
||||
# date: 2026-11-07 15:16:49, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py retry exhausted-replay permanent dlq
|
||||
# exit code: 0, 45.1 s
|
||||
|
||||
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
|
||||
2 x storage_unavailable -> retries after 1.20 s and 2.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
|
||||
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
|
||||
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
|
||||
ops replay -> same job_id 44cac72b-36f8-411d-9b08-cb90fd27bbb9 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
|
||||
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
|
||||
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
|
||||
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
|
||||
schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
|
||||
# date: 2026-11-07 15:12:07, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_jobs.py variants
|
||||
# exit code: 0, 9.6 s
|
||||
|
||||
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
|
||||
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,26 @@
|
||||
# lab7-security: lab7: HTTPS only, headers, outsider ACL, logout, shared login rate limit, size limits, Argon2id, rehash, no credential leaks
|
||||
# date: 2026-11-07 15:07:42, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_lab7_security.py
|
||||
# exit code: 0, 3.8 s
|
||||
|
||||
port 8080: something else answers locally, not the API
|
||||
empty trust store: rejected by certificate verification (unable to get local issuer certificate)
|
||||
foreign CA: rejected by certificate verification (unable to get local issuer certificate)
|
||||
wrong host name: rejected by certificate verification (Hostname mismatch, certificate is not valid for 'evil.example'.)
|
||||
PASS: only HTTPS on 8443; untrusted CA and wrong host name fail certificate verification
|
||||
HSTS, nosniff, no Server header; X-Instance-Id is an opaque alias (83d68b5bf960)
|
||||
PASS: security headers
|
||||
outsider: 6 attempts on a foreign chat/attachment -> 404 (existence not revealed); X-User-Id of the victim is ignored
|
||||
PASS: ACL — outsider reads/writes nothing, cannot attach a foreign file
|
||||
PASS: logout -> the same token is 401 on every replica (session deleted in shared Redis)
|
||||
codes [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429, 429, 429]; answered by replicas ['83d68b5bf960', 'd7b47c3fc484']; Retry-After 60 s
|
||||
PASS: login rate limit is shared by all replicas (Redis), 429 + Retry-After, before Argon2id
|
||||
JSON > 64 KiB -> 413 (with Content-Length and chunked); upload Content-Length 50 MB -> 413 at the LB
|
||||
PASS: request size limits
|
||||
stored: $argon2id$v=19$m=32768,t=2,p=1$<16-byte salt>$<hash>; two users with the same password have different hashes
|
||||
PASS: Argon2id with salt and parameters in the stored string
|
||||
legacy hash m=12288,t=3 -> after successful login m=32768,t=2,p=1; the new hash still verifies
|
||||
PASS: old password hash is upgraded on successful login
|
||||
errors do not echo credentials; Redis stores session:<SHA-256(token)> only; DB stores only the hash
|
||||
PASS: credentials and session tokens are not revealed
|
||||
9 passed, 0 failed
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"container_logs": {
|
||||
"password": 0,
|
||||
"token": 0,
|
||||
"basic_b64": 0
|
||||
},
|
||||
"container_logs_bearer_or_basic_header": 0,
|
||||
"loki": "unavailable: URLError",
|
||||
"tempo": {
|
||||
"traces_fetched": 0,
|
||||
"password": 0,
|
||||
"token": 0,
|
||||
"basic_b64": 0
|
||||
},
|
||||
"prometheus_labels": "unavailable: URLError",
|
||||
"otlp_payloads": {
|
||||
"password": 0,
|
||||
"token": 0,
|
||||
"basic_b64": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# leak-check: lab7: password/token/Authorization absent in container logs, Loki, Tempo traces, Prometheus labels
|
||||
# date: 2026-11-07 15:08:02, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/leak_check.py
|
||||
# exit code: 0, 20.6 s
|
||||
|
||||
{"container_logs": {"password": 0, "token": 0, "basic_b64": 0}, "container_logs_bearer_or_basic_header": 0, "loki": "unavailable: URLError", "tempo": {"traces_fetched": 0, "password": 0, "token": 0, "basic_b64": 0}, "prometheus_labels": "unavailable: URLError", "otlp_payloads": {"password": 0, "token": 0, "basic_b64": 0}}
|
||||
searched: container logs, Tempo, raw OTLP payloads (741 KiB); traces fetched: 0
|
||||
PASS: no password, session token or Authorization value in logs or telemetry
|
||||
@@ -0,0 +1,749 @@
|
||||
[
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "INSERT outbox (enqueue)",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "UPDATE users.username",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "SELECT outbox",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table outbox"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "DELETE messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "DROP TABLE messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "must be owner of table messages"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "CREATE TABLE",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for schema public"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "postgres",
|
||||
"action": "ALTER ROLE (escalate)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied to alter role"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "PUT staging/",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "DELETE staging/ (rollback)",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "PUT attachments/ (worker's prefix)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "GET staging/ (upload sources)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "LIST bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "DELETE attachments/",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "s3",
|
||||
"action": "CREATE bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password', 'redis_password', 's3_access_key_id', 's3_secret_access_key']"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/worker",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/dispatcher",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/janitor",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "GET session:*",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "GET other key",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "this user has no permissions to access one of the keys used as arguments"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "KEYS *",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "this user has no permissions to run the 'keys' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "FLUSHALL",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "this user has no permissions to run the 'flushall' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "CONFIG GET",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "this user has no permissions to run the 'config|get' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "ACL LIST",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "this user has no permissions to run the 'acl|list' command"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"area": "redis",
|
||||
"action": "SUBSCRIBE / PUBLISH",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "this user has no permissions to run the 'publish' command"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "SELECT attachments",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users (passwords)",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "SELECT messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "DELETE attachments",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table attachments"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "postgres",
|
||||
"action": "UPDATE outbox",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table outbox"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "PUT attachments/",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "LIST attachments/",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "LIST bucket root",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "PUT outside known prefixes",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "s3",
|
||||
"action": "CREATE bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/api",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/dispatcher",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/janitor",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "SELECT outbox",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "SELECT messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "postgres",
|
||||
"action": "DELETE attachments",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table attachments"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password']"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/api",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/worker",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/janitor",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "SELECT attachments",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "SELECT users",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table users"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "INSERT messages",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table messages"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "postgres",
|
||||
"action": "DELETE chats",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "permission denied for table chats"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "s3",
|
||||
"action": "LIST bucket",
|
||||
"expect": "allowed",
|
||||
"observed": "allowed",
|
||||
"pass": true,
|
||||
"detail": ""
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "s3",
|
||||
"action": "GET object content",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "s3",
|
||||
"action": "CREATE bucket",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "AccessDenied"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "own secret keys (no admin credentials)",
|
||||
"expect": "ok",
|
||||
"observed": "ok",
|
||||
"pass": true,
|
||||
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/api",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/worker",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/dispatcher",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/migrate",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "read messenger/s3-init",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "write own secret",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/messenger/data/janitor -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "write ACL policy",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "list auth methods",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
|
||||
},
|
||||
{
|
||||
"service": "janitor",
|
||||
"area": "openbao",
|
||||
"action": "create token",
|
||||
"expect": "denied",
|
||||
"observed": "denied",
|
||||
"pass": true,
|
||||
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,115 @@
|
||||
# least-privilege: lab7: each service identity vs PostgreSQL grants, S3 prefixes, OpenBao policies, Redis ACL
|
||||
# date: 2026-11-07 15:07:38, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/test_least_privilege.py
|
||||
# exit code: 0, 2.7 s
|
||||
|
||||
|
||||
api -> postgres
|
||||
PASS SELECT users expect allowed observed allowed
|
||||
PASS INSERT outbox (enqueue) expect allowed observed allowed
|
||||
PASS UPDATE users.username expect denied observed denied
|
||||
PASS SELECT outbox expect denied observed denied
|
||||
PASS DELETE messages expect denied observed denied
|
||||
PASS DROP TABLE messages expect denied observed denied
|
||||
PASS CREATE TABLE expect denied observed denied
|
||||
PASS ALTER ROLE (escalate) expect denied observed denied
|
||||
|
||||
api -> s3
|
||||
PASS PUT staging/ expect allowed observed allowed
|
||||
PASS DELETE staging/ (rollback) expect allowed observed allowed
|
||||
PASS PUT attachments/ (worker's prefix) expect denied observed denied
|
||||
PASS GET staging/ (upload sources) expect denied observed denied
|
||||
PASS LIST bucket expect denied observed denied
|
||||
PASS DELETE attachments/ expect denied observed denied
|
||||
PASS CREATE bucket expect denied observed denied
|
||||
|
||||
api -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/worker expect denied observed denied
|
||||
PASS read messenger/dispatcher expect denied observed denied
|
||||
PASS read messenger/janitor expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
api -> redis
|
||||
PASS GET session:* expect allowed observed allowed
|
||||
PASS GET other key expect denied observed denied
|
||||
PASS KEYS * expect denied observed denied
|
||||
PASS FLUSHALL expect denied observed denied
|
||||
PASS CONFIG GET expect denied observed denied
|
||||
PASS ACL LIST expect denied observed denied
|
||||
PASS SUBSCRIBE / PUBLISH expect denied observed denied
|
||||
|
||||
worker -> postgres
|
||||
PASS SELECT attachments expect allowed observed allowed
|
||||
PASS SELECT users (passwords) expect denied observed denied
|
||||
PASS SELECT messages expect denied observed denied
|
||||
PASS DELETE attachments expect denied observed denied
|
||||
PASS UPDATE outbox expect denied observed denied
|
||||
|
||||
worker -> s3
|
||||
PASS PUT attachments/ expect allowed observed allowed
|
||||
PASS LIST attachments/ expect allowed observed allowed
|
||||
PASS LIST bucket root expect denied observed denied
|
||||
PASS PUT outside known prefixes expect denied observed denied
|
||||
PASS CREATE bucket expect denied observed denied
|
||||
|
||||
worker -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/api expect denied observed denied
|
||||
PASS read messenger/dispatcher expect denied observed denied
|
||||
PASS read messenger/janitor expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
dispatcher -> postgres
|
||||
PASS SELECT outbox expect allowed observed allowed
|
||||
PASS SELECT users expect denied observed denied
|
||||
PASS SELECT messages expect denied observed denied
|
||||
PASS DELETE attachments expect denied observed denied
|
||||
|
||||
dispatcher -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/api expect denied observed denied
|
||||
PASS read messenger/worker expect denied observed denied
|
||||
PASS read messenger/janitor expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
janitor -> postgres
|
||||
PASS SELECT attachments expect allowed observed allowed
|
||||
PASS SELECT users expect denied observed denied
|
||||
PASS INSERT messages expect denied observed denied
|
||||
PASS DELETE chats expect denied observed denied
|
||||
|
||||
janitor -> s3
|
||||
PASS LIST bucket expect allowed observed allowed
|
||||
PASS GET object content expect denied observed denied
|
||||
PASS CREATE bucket expect denied observed denied
|
||||
|
||||
janitor -> openbao
|
||||
PASS own secret keys (no admin credentials) expect ok observed ok
|
||||
PASS read messenger/api expect denied observed denied
|
||||
PASS read messenger/worker expect denied observed denied
|
||||
PASS read messenger/dispatcher expect denied observed denied
|
||||
PASS read messenger/migrate expect denied observed denied
|
||||
PASS read messenger/s3-init expect denied observed denied
|
||||
PASS write own secret expect denied observed denied
|
||||
PASS write ACL policy expect denied observed denied
|
||||
PASS list auth methods expect denied observed denied
|
||||
PASS create token expect denied observed denied
|
||||
|
||||
83 checks, 0 failed; saved to evidence/least-privilege.json
|
||||
PASS: every service identity can do its job and nothing else that was probed
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"old_serial": "7867bb9789172458",
|
||||
"new_serial": "aed4162e0d6ff182",
|
||||
"probe": {
|
||||
"valid LB identity (CN=lb)": "http_200",
|
||||
"foreign CA, CN=lb": "connection_closed_after_handshake",
|
||||
"our CA, expired, CN=lb": "connection_closed_after_handshake",
|
||||
"previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"
|
||||
},
|
||||
"api_events_after_revoke": {
|
||||
"tls.crl_loaded": 2,
|
||||
"tls.revoked_connections_closed": 2,
|
||||
"tls.client_rejected": 4
|
||||
},
|
||||
"load": {
|
||||
"seconds": 21.3,
|
||||
"codes": {
|
||||
"me:200": 265,
|
||||
"send:201": 265,
|
||||
"login:201": 10
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-rotate-leaf: lab7: LB leaf rotated (HAProxy reload), old leaf revoked via CRL, rejected by API
|
||||
# date: 2026-11-07 15:08:49, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/lifecycle.py rotate-leaf
|
||||
# exit code: 0, 21.8 s
|
||||
|
||||
--- rotate-leaf: клиентский сертификат LB (CN=lb)
|
||||
{"old_serial": "7867bb9789172458", "new_serial": "aed4162e0d6ff182", "probe": {"valid LB identity (CN=lb)": "http_200", "foreign CA, CN=lb": "connection_closed_after_handshake", "our CA, expired, CN=lb": "connection_closed_after_handshake", "previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"}, "api_events_after_revoke": {"tls.crl_loaded": 2, "tls.revoked_connections_closed": 2, "tls.client_rejected": 4}, "load": {"seconds": 21.3, "codes": {"me:200": 265, "send:201": 265, "login:201": 10}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors
|
||||
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-rotate-leaf.json
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"kv_version": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"replicas": 2,
|
||||
"active_passwords_during_overlap": 2,
|
||||
"all_replicas_picked_up_after_s": 4.2,
|
||||
"active_password_hashes_after": 1,
|
||||
"new_password_hash_matches": true,
|
||||
"load": {
|
||||
"seconds": 17.2,
|
||||
"codes": {
|
||||
"me:200": 201,
|
||||
"send:201": 201,
|
||||
"login:201": 8
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-rotate-secret: lab7: Redis password of api rotated via OpenBao under load, old one revoked
|
||||
# date: 2026-11-07 15:08:27, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/lifecycle.py rotate-secret
|
||||
# exit code: 0, 17.5 s
|
||||
|
||||
--- rotate-secret: пароль Redis ACL-пользователя api
|
||||
{"kv_version": [1, 2], "replicas": 2, "active_passwords_during_overlap": 2, "all_replicas_picked_up_after_s": 4.2, "active_password_hashes_after": 1, "new_password_hash_matches": true, "load": {"seconds": 17.2, "codes": {"me:200": 201, "send:201": 201, "login:201": 8}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors
|
||||
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-rotate-secret.json
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"sealed_seconds": 73.7,
|
||||
"restarted_replica_health_while_sealed": "unhealthy",
|
||||
"replicas_answering_while_sealed": 1,
|
||||
"restarted_replica_ready_after_unseal_s": 3.2,
|
||||
"api_events": {
|
||||
"secrets.refresh_failed": 25,
|
||||
"secrets.waiting": 8,
|
||||
"secrets.login": 2
|
||||
},
|
||||
"load": {
|
||||
"seconds": 85.2,
|
||||
"codes": {
|
||||
"me:200": 1108,
|
||||
"send:201": 1108,
|
||||
"login:201": 41
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-sealed: lab7: sealed OpenBao: running replicas serve, restarted replica not ready until unseal
|
||||
# date: 2026-11-07 15:11:57, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/lifecycle.py sealed
|
||||
# exit code: 0, 85.6 s
|
||||
|
||||
--- sealed: OpenBao запечатан
|
||||
{"sealed_seconds": 73.7, "restarted_replica_health_while_sealed": "unhealthy", "replicas_answering_while_sealed": 1, "restarted_replica_ready_after_unseal_s": 3.2, "api_events": {"secrets.refresh_failed": 25, "secrets.waiting": 8, "secrets.login": 2}, "load": {"seconds": 85.2, "codes": {"me:200": 1108, "send:201": 1108, "login:201": 41}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready (fail closed) until unseal
|
||||
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-sealed.json
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"role_ttl": "20s / max 40s",
|
||||
"observed_seconds": 100,
|
||||
"api_events": {
|
||||
"secrets.refresh_failed": 2,
|
||||
"secrets.login": 6,
|
||||
"secrets.renewed": 16
|
||||
},
|
||||
"load": {
|
||||
"seconds": 102.1,
|
||||
"codes": {
|
||||
"me:200": 1302,
|
||||
"send:201": 1302,
|
||||
"login:201": 49
|
||||
},
|
||||
"errors_5xx_or_conn": 0,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# lifecycle-token-expiry: lab7: OpenBao token TTL 20 s / max 40 s + revoke: services re-login with their cert
|
||||
# date: 2026-11-07 15:10:32, commit: 5bfc53a + uncommitted changes
|
||||
# command: python student-tests/lifecycle.py token-expiry
|
||||
# exit code: 0, 102.6 s
|
||||
|
||||
--- token-expiry: короткий TTL токенов OpenBao роли api
|
||||
{"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": {"secrets.refresh_failed": 2, "secrets.login": 6, "secrets.renewed": 16}, "load": {"seconds": 102.1, "codes": {"me:200": 1302, "send:201": 1302, "login:201": 49}, "errors_5xx_or_conn": 0, "errors": []}}
|
||||
PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected
|
||||
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-token-expiry.json
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 7,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.028,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 4538,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 151.124,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.062,
|
||||
"p95": 42.691,
|
||||
"p99": 84.746
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user