Author SHA1 Message Date
kalibr456 cb38fd5231 лр7: Docker-прогон, evidence, исправления 2026-11-07 20:17:11 +03:00
249 changed files with 114500 additions and 82 deletions
+61 -25
View File
@@ -1,41 +1,77 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# lab7: здесь НЕТ секретов и путей к файлам с паролями. Учётные данные процессов (PostgreSQL, Redis, S3)
# выдаёт OpenBao по сертификату сервиса; адреса, роли и TLS-пути задаёт compose.yaml для каждого сервиса.
# Соответствие переменных skeleton -> решение — REPORT.md, «Соответствие .env.example».
APP_HOST=0.0.0.0
APP_PORT=8080
APP_PORT=8443
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
SESSION_STORE=redis
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=https://s3:9000
PUBLIC_BASE_URL=https://localhost:8443
# S3 (SeaweedFS 4.52): HTTPS, у api/worker/janitor свои ключи с правами по prefix (из OpenBao)
S3_ENDPOINT=https://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
# lab5: broker/worker. Брокер — amqps://rabbitmq:5671/messenger, вход по сертификату (EXTERNAL); задаётся в compose.
WORKER_CONCURRENCY=2
WORKER_PREFETCH=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=15
JOB_BACKOFF_BASE_SECONDS=2
JOB_BACKOFF_MAX_SECONDS=60
WORKER_MEMORY_LIMIT_MB=512
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
PUBLIC_BASE_URL=https://localhost:8443
TLS_CA_FILE=/run/secrets/ca.crt
TLS_CERT_FILE=/run/secrets/service.crt
TLS_KEY_FILE=/run/secrets/service.key
BAO_ADDR=https://openbao:8200
BAO_CACERT=/run/secrets/bao-ca.crt
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
ARGON2_MEMORY_KIB=19456
PIPELINE_VERSION=1
# Pools and timeouts (как в lab3–6)
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_POOL_WAIT_SECONDS=3
DB_CONNECT_TIMEOUT_SECONDS=3
DB_STATEMENT_TIMEOUT_MS=5000
REDIS_POOL_MAX=20
REDIS_TIMEOUT_MS=500
SHUTDOWN_DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=10
S3_CONNECT_TIMEOUT_SECONDS=1
S3_READ_TIMEOUT_SECONDS=5
S3_MAX_ATTEMPTS=2
S3_POOL_MAX=20
UPLOAD_TIMEOUT_SECONDS=60
UPLOAD_IDLE_TIMEOUT_SECONDS=10
DECODE_CONCURRENCY=2
ATTACHMENT_GRACE_SECONDS=3600
JANITOR_INTERVAL_SECONDS=300
# lab6: OpenTelemetry (service.name и mTLS-сертификат для каждого сервиса задаёт compose.yaml)
OTEL_SDK_DISABLED=false
# lab7: OpenBao. Адрес/CA/роль задаёт compose.yaml (BAO_ADDR=https://openbao:8200, BAO_ROLE=<CN сервиса>).
# Как часто процесс перечитывает свои секреты (подхват ротации) и продлевает токен.
SECRETS_REFRESH_SECONDS=30
SECRETS_STARTUP_TIMEOUT_SECONDS=60
# mTLS: CRL учебного CA перечитывается API раз в N секунд (отзыв identity без рестарта).
CRL_RELOAD_SECONDS=5
# lab7: Argon2id (не ниже m=19456 KiB, t=2, p=1). Выбор 32 MiB / t=2 / p=1 — по замеру, REPORT.md.
ARGON2_MEMORY_KIB=32768
ARGON2_ITERATIONS=2
ARGON2_PARALLELISM=1
# Бюджет хеширования реплики: не больше N хешей одновременно, ожидание слота не дольше — иначе 503 + Retry-After.
ARGON2_MAX_CONCURRENCY=2
ARGON2_QUEUE_SECONDS=2
# lab7: общий (Redis) лимит попыток входа/регистрации, фиксированное окно.
LOGIN_RATE_WINDOW_SECONDS=60
LOGIN_RATE_PER_USERNAME=10
LOGIN_RATE_PER_CLIENT=600
REGISTER_RATE_PER_CLIENT=600
# Ограничение размера JSON-тела (вложения — MAX_UPLOAD_BYTES).
MAX_JSON_BYTES=65536
+23
View File
@@ -0,0 +1,23 @@
*.sh text eol=lf
*.sql text eol=lf
*.conf text eol=lf
*.cfg text eol=lf
*.example text eol=lf
*.cmd text eol=crlf
*.yaml text eol=lf
*.yml text eol=lf
*.json text eol=lf
*.py text eol=lf
*.md text eol=lf
# Конфиги, которые читают Linux-контейнеры (OpenBao, SeaweedFS, RabbitMQ), и файлы сборки
*.hcl text eol=lf
*.toml text eol=lf
*.txt text eol=lf
*.http text eol=lf
infra/rabbitmq-enabled-plugins text eol=lf
Dockerfile text eol=lf
Makefile text eol=lf
.gitignore text eol=lf
.dockerignore text eol=lf
*.jpg binary
*.png binary
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 7
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Бабиченко Максим, БВТ2303
Целевая оценка (3/4/5): 4
Трек 5, если заявлен: не заявлен
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: перенесено всё решение lab6 (API lab1–5, outbox/dispatcher/worker/janitor, OpenTelemetry + Collector/Prometheus/Loki/Tempo/Grafana, тесты lab1–6). Добавлены: учебный PKI оператора (`scripts/pki.py`: CA, leaf на каждую identity, CRL, ротация/отзыв); HTTPS-ingress и mTLS LB→API с allowlist CN=lb и CRL (в т.ч. закрытие открытых соединений отозванных сертификатов); TLS на всех связях — PostgreSQL (verify-full, hostssl+SCRAM, 4 роли с GRANT), Redis (TLS+client cert+ACL), S3 (HTTPS, ключи по prefix, mTLS внутреннего gRPC), RabbitMQ (AMQPS, EXTERNAL по CN), Collector (mTLS-шлюз с allowlist CN), Prometheus/Loki/Tempo/Grafana (mTLS); OpenBao 2.6.2 не dev-mode (file storage, init/unseal оператором, cert auth по CN, политики по пути, токены 15 мин/1 ч, renew/relogin, работа при sealed); Argon2id 32 MiB/t=2/p=1 с rehash, бюджет хеширования на реплику, общий rate limit в Redis (429 + Retry-After), лимиты тела; hardening всех контейнеров; скан зависимостей (Pillow → 13.0.0) и образов.
Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`pki.py init`, `init_secrets.py`, `docker compose up -d openbao`, `bao.py init/unseal/configure`, `docker compose up -d --build --wait`, `collect_evidence.py`, `scan.py`).
Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев»; `evidence/native-sandbox/` (прогон без Docker), `evidence/` (Docker-прогон), `contracts/trust-matrix.md`, `security/THREAT_MODEL.md`, `security/scan-triage.md`.
Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги» (один хост, доли unseal в файле оператора, CRL не проверяют RabbitMQ/Redis/PostgreSQL, Loki/Tempo не разделяют читателя и писателя).
Источники и использование ИИ: REPORT.md, шапка (документация OWASP, OpenBao, RabbitMQ, PostgreSQL, Redis, SeaweedFS, Prometheus, Grafana; ИИ-ассистент Claude).
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод (песочница: `evidence/native-sandbox/`; Docker-прогон: `evidence/`).
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены (Docker-прогон: `evidence/*.json`).
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+42
View File
@@ -0,0 +1,42 @@
# Образ приложения lab7 (api, worker, dispatcher, janitor, migrate, s3-init, ops.py — разные команды).
# В образе нет ни одного секрета/ключа: сертификаты и CA монтируются при запуске (secrets/pki/*),
# учётные данные процесс получает из OpenBao по своему сертификату.
FROM python:3.11.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
WORKDIR /app
# Security-обновления Debian базового слоя: скан Docker-прогона (security/scan-triage.md) нашёл в python:3.11.13-slim
# OpenSSL 3.5.1 (CRITICAL/HIGH), zlib, sqlite, libuuid — их загружает сам CPython (ssl, zlib, uuid) на каждом TLS-соединении.
RUN apt-get update && apt-get upgrade -y --no-install-recommends && rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
# После установки зависимостей pip/setuptools/wheel не нужны в рантайме — удаляем (меньше поверхность и находок скана).
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt \
&& (pip uninstall -y pip setuptools wheel || true)
COPY main.py settings.py errors.py db.py dbconn.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py \
s3_init.py pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py telemetry.py \
secretstore.py tlsutil.py ./
COPY migrations/ ./migrations/
# Фиксированный uid/gid (10001): не root; файлы кода принадлежат root и только читаются.
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin app \
&& mkdir -p /faults && chown app:app /faults
# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher).
USER 10001:10001
ARG APP_VERSION=dev
ENV APP_VERSION=${APP_VERSION}
EXPOSE 8443
# Проверка здоровья — loopback listener 127.0.0.1:8081 (только /health/*), бизнес-API там нет.
HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=4)"
CMD ["python", "main.py"]
+304 -16
View File
@@ -1,37 +1,325 @@
# Отчёт — лабораторная 7
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Бабиченко Максим, БВТ2303
- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не выбран
- Commit/PR: <<ссылка на MR после публикации>>
- Использованные источники, библиотеки и помощь ИИ: всё из lab6 (FastAPI, psycopg 3, pika, boto3, Pillow, OpenTelemetry SDK, RabbitMQ 4.1, SeaweedFS, PostgreSQL 17, Redis 7.4, HAProxy 3.2, OTel Collector, Prometheus, Loki, Tempo, Grafana). Новое: `cryptography` 50.0.2 (выпуск сертификатов оператором, разбор и проверка подписи CRL в API), образ `openbao/openbao:2.6.2`, SeaweedFS обновлён до 4.52 (TLS шлюза S3, права по prefix, mTLS внутреннего gRPC), Pillow 12.3.0 → 13.0.0 и security-обновления Debian в образе приложения (скан). Документация: OWASP Password Storage и File Upload Cheat Sheets, RFC 9106 (Argon2), OpenBao (cert auth, policies, KV v2, seal/unseal, production hardening), RabbitMQ TLS и `rabbitmq_auth_mechanism_ssl`, PostgreSQL `pg_hba`/SSL/SCRAM, Redis TLS и ACL, SeaweedFS `security.toml`, Prometheus web configuration (`client_allowed_sans`), Grafana/Tempo/Loki TLS, Docker security, pip-audit, Trivy. При написании кода, конфигураций и тестов использовался ИИ-ассистент (Claude); весь код я прогнал и могу объяснить и изменить на защите.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Проверяю на Windows (PowerShell, Docker Desktop). На хосте нужен Python 3.11+: тесты — только стандартная библиотека, операторские скрипты PKI — пакет `cryptography` (`requirements-ops.txt`). Перед запуском остановить стенд lab6 (`docker compose down` в его папке). Имя проекта Compose — `messenger-lab7`, volumes свои.
```powershell
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab7.git
cd mtusi2026lab7
git checkout solution/bvt2303-babichenko
$env:PYTHONUTF8 = "1"
python -m pip install -r requirements-ops.txt # cryptography — только для scripts/pki.py
Copy-Item .env.example .env
# 1. Оператор: учебный CA и сертификаты сервисов (ключ CA остаётся в secrets/pki/ca, никуда не монтируется)
python scripts/pki.py init
# 2. Оператор: пароли/ключи стенда (серверные конфиги + файл для OpenBao), значения не печатаются
python scripts/init_secrets.py
# 3. OpenBao: старт, инициализация (3 доли, порог 2), распечатка, настройка (KV, cert auth, политики, секреты)
docker compose up -d openbao
python scripts/bao.py init
python scripts/bao.py unseal
python scripts/bao.py configure
# 4. Весь стенд
docker compose --env-file .env config --quiet
docker compose up -d --build --wait
docker compose ps -a # migrate, s3-init, telemetry-init: Exited (0); остальные running/healthy
# 5. Проверки
python tests/smoke.py --base-url https://localhost:8443 --ca-file secrets/pki/public/ca.crt # make test CA_FILE=...
python tests/security.py --base-url https://localhost:8443 --ca-file secrets/pki/public/ca.crt # make test-security
python student-tests/collect_evidence.py # все шаги lab7 + регрессия lab1–6 -> evidence/, таблицы в REPORT.md
python scripts/scan.py # pip-audit + Trivy по образам -> evidence/scan/
```
| Что | Где |
| --- | --- |
| API (единственная точка входа) | https://localhost:8443, доверять `secrets/pki/public/ca.crt` (сертификат ingress: SAN localhost, 127.0.0.1) |
| Grafana | https://localhost:3000, логин `admin`, пароль — `secrets/grafana_admin_password`. Браузеру нужен наш CA: `Import-Certificate -FilePath secrets\pki\public\ca.crt -CertStoreLocation Cert:\CurrentUser\Root` (по желанию; проверку не отключаем) |
| Prometheus / Loki / Tempo | https://localhost:9090, :3100, :3200 — только с клиентским сертификатом оператора: `curl.exe --cacert secrets/pki/public/ca.crt --cert secrets/pki/operator/tls.crt --key secrets/pki/operator/tls.key https://localhost:9090/-/ready` |
| Статистика HAProxy | https://localhost:8404/stats — тот же сертификат оператора |
| OpenBao (оператор) | https://127.0.0.1:8200, `python scripts/bao.py status` |
Отдельные проверки lab7 (make на Windows нет):
| Команда | Что проверяет |
| --- | --- |
| `python student-tests/test_tls_matrix.py` | вся trust matrix изнутри сети (контейнер `probe`): разрешённая identity проходит, без сертификата / чужой CA / чужая identity нашего CA / истёкший / отозванный — отказ; plaintext не отвечает |
| `python student-tests/test_least_privilege.py` | api/worker/dispatcher/janitor своими учётными данными: GRANT PostgreSQL, prefix S3, политики OpenBao, ACL Redis |
| `python student-tests/test_lab7_security.py` | только HTTPS, заголовки, посторонний пользователь, logout, общий rate limit, лимиты тела, Argon2id, rehash, отсутствие утечек |
| `python student-tests/lifecycle.py rotate-secret rotate-leaf token-expiry sealed` | ротация секрета и leaf, отзыв, истечение токенов, sealed OpenBao — под нагрузкой |
| `python student-tests/leak_check.py` | пароль/токен/Authorization не попадают в логи контейнеров, Loki, Tempo, метки Prometheus |
| `python student-tests/argon2_bench.py`, `python student-tests/login_dos.py` | замер Argon2id в контейнере api; бюджет защиты от login DoS |
| `python student-tests/check_hardening.py --runtime` | privileged/host/docker.sock, capabilities, no-new-privileges, read_only, не root, лимиты, порты |
| `python student-tests/run_persistence.py`, `python student-tests/run_failover.py` | публичные `test-persistence` (hook `scripts\restart.cmd`, включая рестарт и unseal OpenBao) и `test-failover` (hook `scripts\stop-one.cmd`) по HTTPS |
**После перезагрузки хоста / рестарта OpenBao** он запечатан: `python scripts/bao.py unseal`. До этого уже работающих процессов нет, новые не стартуют (ждут до 60 с и перезапускаются Docker) — после unseal стенд поднимается сам. **Остановка без потери данных:** `docker compose down` (volumes `openbaodata`, `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults`, `promdata`, `lokidata`, `tempodata`, `grafanadata` остаются). **Намеренное удаление учебных данных:** `docker compose down -v`, удалить папку `secrets` (CA, ключи, доли unseal, пароли) и пройти шаги 1–4 заново.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
клиент --HTTPS 127.0.0.1:8443--> lb (HAProxy) ==mTLS, клиент CN=lb==> api x2 (FastAPI)
api --TLS verify-full + SCRAM, роль messenger_api--> postgres api --TLS+mTLS + ACL user api--> redis
api --HTTPS + ключ api (staging/ write, attachments/ read)--> s3 api ==cert auth CN=api==> openbao
api/worker/dispatcher/janitor ==mTLS, CN из allowlist==> otel-gate --127.0.0.1--> otel-collector
dispatcher ==AMQPS EXTERNAL CN=dispatcher==> rabbitmq <==AMQPS EXTERNAL CN=worker== worker
otel-collector ==mTLS CN=otel-collector==> prometheus | loki | tempo <==mTLS CN=grafana== grafana
```
**Что перенесено из lab6:** весь API (lab1–5), outbox/dispatcher/worker/janitor, телеметрия (SDK, Collector, Prometheus/Loki/Tempo/Grafana, dashboard, алерты, runbooks), тесты lab1–6. **Что изменено в lab7:**
| Область | Решение | Где |
| --- | --- | --- |
| PKI | Учебный CA оператора (ECDSA P-256), leaf на каждую identity (30 суток, CN + URI SAN `spiffe://messenger.lab/<cn>`), CRL, ротация/отзыв, тестовые сертификаты (чужой CA, intruder, истёкший) | `scripts/pki.py` |
| Внешний TLS | HAProxy: только HTTPS :8443, TLS 1.2+, без tickets, HSTS, nosniff, без `Server`; `X-Forwarded-For` перезаписывается; тело > 11 MiB → 413 | `infra/haproxy.cfg` |
| LB → API | mTLS: HAProxy проверяет сертификат API (CA, CRL, имя api) и предъявляет `lb`; API (`tlsutil.py`, свой протокол uvicorn) — цепочка/срок (OpenSSL) + **allowlist CN** и CRL до чтения запроса; CRL перечитывается раз в 5 с, открытые соединения отозванных сертификатов закрываются. Health — отдельный listener 127.0.0.1:8081 | `tlsutil.py`, `main.py` |
| OpenBao | Не dev-mode: file storage в volume, TLS-listener, init/unseal оператором, KV v2 `messenger/`, **cert auth** (роль = CN, только наш CA, CRL), 6 политик по одному пути, токены 15 мин / max 1 ч. Клиент приложения: вход, renew, relogin, перечитывание KV раз в 30 с, работа на кэше при sealed | `infra/openbao/`, `scripts/bao.py`, `secretstore.py` |
| PostgreSQL | `ssl=on`, `pg_hba`: только `hostssl` + SCRAM; 4 роли приложения с GRANT по запросам (`006_roles.sql`), пароли из OpenBao на каждое соединение, migrate ставит их SCRAM-verifier'ом | `infra/postgres/pg_hba.conf`, `dbconn.py`, `migrate.py` |
| Redis | Только TLS, клиентский сертификат обязателен, ACL: `default off`, `api` — `session:*`/`ratelimit:*` и 13 команд; пароль через credential provider (ротация без рестарта) | `infra/redis.conf`, `auth.py` |
| S3 | SeaweedFS 4.52: HTTPS :8333, внутренние порты на loopback, gRPC :18333 — mTLS только CN=s3, IAM/Iceberg/Lance выключены; ключи api/worker/janitor/s3-init с правами по prefix | `infra/seaweedfs/security.toml`, `scripts/init_secrets.py` |
| RabbitMQ | Только AMQPS :5671, `verify_peer`, EXTERNAL (пользователь = CN), пользователи без паролей, distribution/epmd на loopback; плагины — только `rabbitmq_auth_mechanism_ssl` | `infra/rabbitmq.conf`, `broker.py` |
| Телеметрия | otel-gate (HAProxy) — mTLS и allowlist CN приложений перед Collector (Collector на 127.0.0.1); Collector → Tempo/Loki/Prometheus по mTLS; Prometheus `client_allowed_sans`; Loki gRPC на loopback, Tempo внутренний gRPC по TLS; Grafana по HTTPS, источники — mTLS | `infra/otel/`, `infra/prometheus/web.yml`, `infra/loki`, `infra/tempo`, `infra/grafana` |
| Пароли и вход | Argon2id m=32 MiB, t=2, p=1 (замер ниже), rehash при входе, ≤2 хеша на реплику одновременно (иначе 503 + Retry-After), общий rate limit в Redis (429 + Retry-After), JSON ≤ 64 KiB (в т.ч. chunked) | `auth.py`, `main.py` |
| Hardening | Все сервисы: cap_drop ALL, no-new-privileges, read_only + tmpfs, не root, лимиты CPU/RAM/PID, нет privileged/host/docker.sock; объяснённые исключения (RabbitMQ записываемый корень, telemetry-init CHOWN); SeaweedFS — uid 1000 образа (в Docker-прогоне исключение uid 0 снято). Наружу — только 127.0.0.1 | `compose.yaml`, `Dockerfile`, `student-tests/check_hardening.py` |
| X-Instance-Id | Случайный alias процесса (12 hex), а не ID контейнера; `stop_one.py` находит реплику по файлу alias | `settings.py`, `scripts/stop_one.py` |
**Источник истины** не менялся: PostgreSQL (данные), S3 (байты), Redis (сессии, счётчики лимитов), OpenBao (учётные данные сервисов). **Транзакционные границы** — как в lab5/6 (outbox). **Готовность (`/health/ready`)**: PostgreSQL + Redis + секреты загружены; OpenBao в readiness не входит — его недоступность не останавливает уже работающую реплику (см. «Эксперименты»).
### Соответствие `.env.example`
| Переменная skeleton | В решении |
| --- | --- |
| `APP_PORT=8080` | API слушает **8443** (только mTLS); наружу — HAProxy 127.0.0.1:8443 |
| `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE`, `S3_*_FILE` | не используются: адреса — `DB_HOST/DB_USER/REDIS_HOST/REDIS_USER/BROKER_URL/S3_ENDPOINT` в compose, учётные данные — из OpenBao (`messenger/<сервис>`); брокер — вход по сертификату |
| `S3_ENDPOINT=https://s3:9000` | `https://s3:8333` (SeaweedFS S3) |
| `OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318` | так и есть (alias otel-gate) + `OTEL_EXPORTER_OTLP_CERTIFICATE/CLIENT_CERTIFICATE/CLIENT_KEY` |
| `PUBLIC_BASE_URL` | `https://localhost:8443` (тесты, ingress) |
| `TLS_CA_FILE`, `TLS_CERT_FILE`, `TLS_KEY_FILE` | `/run/tls-ca/ca.crt`, `/run/tls/tls.crt`, `/run/tls/tls.key` (+ `TLS_CRL_FILE=/run/tls-ca/crl.pem`) |
| `BAO_ADDR`, `BAO_CACERT` | `BAO_ADDR=https://openbao:8200`; CA — тот же `TLS_CA_FILE`; роль — `BAO_ROLE` = CN сертификата |
| `ARGON2_MEMORY_KIB=19456`, `ARGON2_ITERATIONS=2`, `ARGON2_PARALLELISM=1` | `32768 / 2 / 1` (+ `ARGON2_MAX_CONCURRENCY=2`, `ARGON2_QUEUE_SECONDS=2`) |
| — | новые: `SECRETS_REFRESH_SECONDS`, `CRL_RELOAD_SECONDS`, `LOGIN_RATE_*`, `REGISTER_RATE_PER_CLIENT`, `MAX_JSON_BYTES` |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
Где проверено: **Docker-стенд** на машине автора (Windows 10, Docker Desktop, 16 CPU / 15 GiB VM): стенд поднят с нуля шагами раздела «Запуск», затем `python student-tests/collect_evidence.py` — **все 46 шагов PASS** (`evidence/summary.md`), результаты в `evidence/*.txt|json`, замеры — блок ниже. Ранее решение проверялось в **нативной песочнице** без Docker (`evidence/native-sandbox/`, оставлена для сравнения): тот же стенд процессами, конфиги из `infra/`, вместо `docker` — подмена, управляющая процессами. Ошибки, которые песочница не поймала и которые исправлены по Docker-прогону, — в «Ограничениях». Отличия песочницы: PostgreSQL 16 вместо 17, Redis 7.0 вместо 7.4, RabbitMQ 4.4 вместо 4.1.4, HAProxy 2.8 вместо 3.2, **HashiCorp Vault 2.1.2 как API-совместимая замена OpenBao 2.6.2** (тот же HTTP API: sys/init, unseal, cert auth, KV v2, policies), Prometheus 2.45 (только web-config TLS), Collector — заглушка OTLP за настоящим otel-gate; Loki/Tempo/Grafana не запускались (их TLS-конфиги проверяются только в Docker).
### Уровень 3
| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker) | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke по HTTPS с проверяемым CA и hostname | `make test CA_FILE=secrets/pki/public/ca.crt` | PASS: lab7 public API scenario (регистрация, вход, чат, сообщения, пагинация, идемпотентность, вложение → thumbnail, logout) | `evidence/smoke.txt` |
| Внешний TLS: недоверенный CA отвергается | `make test-security`; `test_lab7_security.py test_https_only_and_certificate_checks` | PASS: пустой trust store, чужой CA → `SSLCertVerificationError`; имя `evil.example` → Hostname mismatch; plaintext на 8443 — без ответа приложения; 8080 — не API | `security.txt`, `lab7-security.txt` |
| Предыдущие возможности через HTTPS | регрессия lab1–5: `test_api`, `test_race`, `test_idempotency`, `test_balancing`, `test_attachments`, `test_consistency`, `test_jobs`, `test_janitor_replay`, `test_failures`, `test_db`, `redis_info`, `explain`, `load` | все PASS (см. сводку) | `evidence/*.txt` |
| Argon2id: соль, параметры, ≥19 MiB/t=2/p=1, обоснование замером | `test_password_storage_argon2id`, `argon2_bench.py` | `$argon2id$v=19$m=32768,t=2,p=1$<16-байтная соль>$…`; одинаковый пароль → разные хеши; замер — «Эксперименты» | `lab7-security.txt`, `argon2-bench.json` |
| LB→API mTLS, отказ без cert / чужой CA / неверная identity; нет обхода | `test_tls_matrix.py lb-api` | lb → 200; без сертификата, чужой CA с CN=lb, истёкший lb, intruder (наш CA), worker, api → соединение закрыто; клиент с чужим CA не принимает сервер; plaintext — нет ответа; порт API не опубликован | `tls-matrix.json` |
| Остальные связи — TLS с проверкой сервера, раздельные учётные данные и минимальные права | `test_tls_matrix.py`, `test_least_privilege.py`, `contracts/trust-matrix.md` | 78 проверок matrix (12 связей, включая Loki/Tempo/Grafana и внутренний gRPC Tempo), 83 проверки прав — все PASS | `tls-matrix.json`, `least-privilege.json` |
| OpenBao не dev-mode, persistent storage, сервисная аутентификация, scoped policy, без root token в приложении | `bao.py init/unseal/configure`; `test_least_privilege.py` (openbao); `run_persistence.py` (рестарт OpenBao + unseal, данные на месте) | cert auth по CN; чужие пути, запись KV/политики, список auth, создание токена → 403; worker как роль api → 400 | `least-privilege.txt`, `tls-matrix.txt`, `persistence.txt` |
| Контейнеры: без privileged/host/docker.sock, app/worker non-root, без лишних capabilities, лимиты; наружу только TLS-ingress и локальные админ-порты; секреты не в Git/образах/телеметрии | `check_hardening.py --runtime`, `check_stack.py`, `leak_check.py`, `.gitignore`/`.dockerignore` | статически 183 PASS, 4 объяснённых исключения, 0 нарушений + `docker inspect` всех контейнеров; маркерные пароль/токен — 0 в логах и OTLP | `hardening.json`, `leak-check.json` |
| 429 + Retry-After, лимит размера, посторонний пользователь, logout/expiry, недоверенный TLS | `test_lab7_security.py`, `test_expiry.py` (lab2) | 401×9 → 429, Retry-After 60, ответы двух реплик; JSON > 64 KiB → 413 (и chunked), upload 50 MB → 413 на LB; посторонний → 404 ×6; logout → 401 | `lab7-security.txt` |
| Сертификаты — локальный документированный bootstrap | `pki.py init` → `init_secrets.py` → `bao.py init/unseal/configure` | чистый Docker-стенд по этим шагам поднят с нуля (новые secrets/, volumes) | раздел «Запуск» |
| Smoke, рестарт и отказ реплики через HTTPS без отключения проверки | `run_persistence.py`, `run_failover.py` | PASS (рестарт 16 сервисов, включая OpenBao + unseal, 82.1 с) / PASS ×2 (0 неудачных записей) | `persistence.txt`, `failover.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Уровень 4
## Эксперименты и отказы
| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker) | Файл доказательства |
| --- | --- | --- | --- |
| mTLS API/worker→broker и →Collector с разными identity; сертификат того же CA не даёт лишних прав | `test_tls_matrix.py broker collector` | broker: worker/dispatcher входят, CN=api/intruder/lb → login refused, worker publish / dispatcher basic_get / declare → 403; Collector: api/worker/dispatcher/janitor → 200, lb/grafana/operator/intruder → 403 | `tls-matrix.json` |
| Пароли/token hashes и ошибки не раскрывают credentials | `test_errors_and_storage_do_not_reveal_credentials`, `leak_check.py` | ошибки без эха пароля/username; в Redis только `session:<SHA-256>`; в логах/телеметрии — 0 маркеров | `lab7-security.txt`, `leak-check.json` |
| Rehash старого хеша при входе | `test_rehash_on_login` | m=12288,t=3 (lab2–6) → после входа m=32768,t=2,p=1, повторный вход работает | `lab7-security.txt` |
| Измеренный бюджет защиты от login DoS; rate limit общий для реплик | `login_dos.py`, `test_login_rate_limit_is_shared` | см. «Эксперименты»: атака получает 503 busy / 429, легитимные запросы — ограниченная задержка | `login-dos.json` |
| Ротация рабочего secret и leaf без пересборки, отзыв старой identity | `lifecycle.py rotate-secret rotate-leaf` | Redis-пароль: v1→v2, обе реплики подхватили, активен только новый, 0 ошибок; leaf lb: rotate → reload → revoke → старый на новом соединении отвергнут, API закрыл 2 соединения со старым leaf, 0 ошибок | `lifecycle-rotate-*.json` |
| Ограниченное поведение при expiry / sealed OpenBao; bootstrap trust и renewal описаны; нет постоянного «секрета для получения секретов» | `lifecycle.py token-expiry sealed`; `contracts/trust-matrix.md` «Bootstrap и эксплуатация» | TTL 20 с/max 40 с + отзыв: повторный вход сертификатом, 2393 ответов, 0 ошибок; sealed 75.6 с: работающая реплика без ошибок, перезапущенная — не ready, после unseal ready через 5.4 с | `lifecycle-token-expiry.json`, `lifecycle-sealed.json` |
| read_only / no-new-privileges / drop caps / tmpfs; исключения stateful объяснены | `check_hardening.py --runtime` | см. выше; исключения — в `EXCEPTIONS` скрипта и в `security/THREAT_MODEL.md` (T12) | `hardening.json` |
| Dependency/image scan с разбором применимости | pip-audit, `trivy config`, `scripts/scan.py` | pip-audit: Pillow 12.3.0 — 39 находок, применимо (worker декодирует недоверенные изображения) → 13.0.0 + `formats=(PNG, JPEG)`, повторно 0. Trivy образов (Docker): OpenSSL 3.5.1 (CRITICAL) и zlib/sqlite/libuuid в базовом `python:3.11.13-slim` — применимо (TLS каждого процесса) → `apt-get upgrade` в Dockerfile, OpenSSL 3.5.7, applicable 37 → 9 (оставшиеся без исправления и недостижимы: модули не импортируются); сторонние образы разобраны по ролям, открыто — RabbitMQ (OpenSSL 3.5.4), OpenBao (Go stdlib), HAProxy (musl/pcre2/zlib) | `security/scan-triage.md`, `evidence/scan/` |
| Модель угроз: ≥6 угроз → контроли → отрицательные тесты | `security/THREAT_MODEL.md` | 14 угроз, у каждой — тест из списка выше | — |
| Upload-worker ограничен по CPU/RAM/времени | `test_jobs.py limits` (lab5), compose: worker 2 CPU / 1536 MiB / 128 PID, `imaging.py` RLIMIT_AS/CPU + таймаут | PASS | `jobs-limits.txt` |
| TLS-аудит всей заявленной матрицы, ACL и отсутствие утечек в telemetry | `test_tls_matrix.py`, `test_lab7_security.py`, `leak_check.py` | все PASS | см. выше |
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
<!-- measurements:begin -->
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-07, commit 20afc2c + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB; lab7: openbao 0,5 CPU / 256 MiB, otel-gate 0,25 CPU / 64 MiB; у всех — лимит PID.
TLS/mTLS-аудит trust matrix изнутри сети (`student-tests/test_tls_matrix.py`, контейнер probe):
| Связь | Проверок | Пройдено |
| --- | --- | --- |
| LB -> API (mTLS, allowlist CN=lb) | 10 | 10 |
| API/worker -> RabbitMQ (mTLS + EXTERNAL, CN = broker user) | 11 | 11 |
| API/worker -> OTel Collector (mTLS via otel-gate, allowlist app CNs) | 13 | 13 |
| API/worker -> PostgreSQL (TLS verify-full, SCRAM, roles) | 4 | 4 |
| API -> Redis (TLS + client cert of our CA + ACL user) | 6 | 6 |
| API/worker -> S3 (HTTPS + per-service keys by prefix) | 6 | 6 |
| Services -> OpenBao (TLS + cert auth by CN + policy) | 7 | 7 |
| Collector/Grafana/operator -> Prometheus (mTLS + allowed SANs) | 7 | 7 |
| Collector/Grafana/operator -> Loki (mTLS, CA) | 4 | 4 |
| Collector/Grafana/operator -> Tempo (mTLS, CA) | 4 | 4 |
| Collector -> Tempo OTLP gRPC :4317 (mTLS) | 3 | 3 |
| Operator -> Grafana (HTTPS) | 3 | 3 |
Минимальные права (`test_least_privilege.py`): 83 проверок, 83 пройдено (разрешённое работает, запрещённое — отказ).
Argon2id внутри контейнера api (`argon2_bench.py`, медиана из 7):
| m, KiB | t | p | hash, мс | verify, мс | входов/с на реплику при 2 параллельных | RAM на хеширование, MiB |
| --- | --- | --- | --- | --- | --- | --- |
| 19456 | 2 | 1 | 12.2 | 12.2 | 163.9 | 38 |
| 19456 | 3 | 1 | 18.3 | 18.5 | 108.1 | 38 |
| 32768 (выбрано) | 2 | 1 | 22.6 | 22.2 | 90.1 | 64 |
| 47104 | 2 | 1 | 34.2 | 34.8 | 57.5 | 92 |
| 65536 | 2 | 1 | 49.8 | 50.4 | 39.7 | 128 |
| 65536 | 3 | 1 | 75.3 | 75.2 | 26.6 | 128 |
Login DoS (`login_dos.py`, 16 потоков неверных входов, 20 с): атака 96.2 попыток/с, коды {'401': 360, '429': 1564}; легитимный клиент p95 34.6 -> 194.1 мс, p99 35.5 -> 243.6 мс, коды {'200': 93, '201': 93}.
Ротация пароля Redis (api) через OpenBao: {"kv_version": [5, 6], "replicas": 2, "active_passwords_during_overlap": 2, "all_replicas_picked_up_after_s": 23.9, "active_password_hashes_after": 1, "new_password_hash_matches": true}; нагрузка 37.6 с, ответы {'me:200': 431, 'send:201': 431, 'login:201': 18}, ошибок 5xx/соединения 0.
Ротация leaf LB + отзыв старого через CRL: {"old_serial": "70ea18ded7f86e5a", "new_serial": "22a71730433d9d42", "probe": {"valid LB identity (CN=lb)": "http_200", "foreign CA, CN=lb": "connection_closed_after_handshake", "our CA, expired, CN=lb": "connection_closed_after_handshake", "previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"}, "api_events_after_revoke": {"tls.crl_loaded": 2, "tls.revoked_connections_closed": 2, "tls.client_rejected": 4}}; нагрузка 23.1 с, ответы {'me:200': 243, 'send:201': 243, 'login:201': 11}, ошибок 5xx/соединения 0.
Истечение/отзыв токенов OpenBao: {"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": {"secrets.refresh_failed": 2, "secrets.login": 4}}; нагрузка 101.6 с, ответы {'me:200': 1172, 'send:201': 1172, 'login:201': 49}, ошибок 5xx/соединения 0.
OpenBao sealed: {"sealed_seconds": 75.6, "restarted_replica_health_while_sealed": "starting", "replicas_answering_while_sealed": 1, "restarted_replica_ready_after_unseal_s": 5.4, "api_events": {"secrets.refresh_failed": 5, "secrets.waiting": 8, "secrets.login": 1}}; нагрузка 89.2 с, ответы {'me:200': 1012, 'send:201': 1012, 'login:201': 43}, ошибок 5xx/соединения 0.
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
| --- | --- | --- | --- | --- |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 94.0 (от stop) | 45.6 (от start) |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 124.5 (от stop) | 45.6 (от start) |
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 122.5 (от LOCK) | 35.1 (от COMMIT блокировки) |
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 85.9 (от stop) | 70.6 (от start) |
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 115.4 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 123.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 5.4 с, backlog 0.0.
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '400': 0.0, '409': 0.0, '404': 0.0, '503': 1.44}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 15.6, 'max': 31.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 50418f127c336af7f2e4dc0190adf7ea: server-спан 5010.0 мс, из них `INSERT` 5001.6 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 2e774b9d5912ba768f3e9881f3c41e80, POST /users): server 47.0 мс, `argon2id hash` 32.2 мс, все спаны БД 6.9 мс. Нагрузка: до 102.1 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 102.4/с без ошибок.
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
| --- | --- | --- | --- |
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 8486 запросов, коды {'201': 8486}, доля успеха 1.0 | heap Collector 32.3 -> max 84.7 MiB (limit 200), контейнер max 103.7 MiB | send/enqueue failed в Collector: {'spans': 2048, 'log_records': 213}; пробные трассы дошли 13/13, потеряны отправленные на [] с |
| `stop otel-collector` на 90 с | 5298 запросов, коды {'201': 5298}, доля успеха 1.0 | api 831.6 -> max 866.4 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 19456} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (5554 спанов за минуту) |
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
| --- | --- | --- | --- | --- | --- |
| 1 | 0c7893e1043d | 2 | 126 | 0 | 5.453 |
| 2 | b7f37f051ba1 | 2 | 125 | 0 | 5.313 |
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.54 / 1.64 | 0c7893e1043d: 5 | 129.5 | 141.0 | 11.5 | 164.8 |
| | | | | b7f37f051ba1: 5 | 126.2 | 139.1 | 12.9 | 164.8 |
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.13 / 1.39 | 0c7893e1043d: 5 | 128.9 | 134.9 | 6.0 | 113.5 |
| | | | | b7f37f051ba1: 5 | 126.7 | 134.4 | 7.7 | 113.9 |
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.64 / 2.25 | 0c7893e1043d: 10 | 125.6 | 144.1 | 18.5 | 174.8 |
| | | | | b7f37f051ba1: 10 | 128.7 | 147.9 | 19.2 | 182.8 |
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 162 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 206.623 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 5.453, 5.313 с (hook ждёт остановки ~4 с); потерь нет |
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 82.1 с до готовности; потерь нет |
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 781 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 8703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6133fe0a-b568-4024-8bb7-4d7d32ed0433 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 60ca1ef4-a5fb-48ad-94bd-5f14f683304d ready, attempt 1, no manual action | без ручного действия |
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 18 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready |
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.70 s and 2.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 3d109dba-1036-482a-a6fb-8116c7894b6d attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (4 message(s)); dlq-replay kept them: ['unknown job', 'unknown job'] | сообщения сохранены |
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.6 s after worker start, peak processing 2 of 8 (6 samples) | - |
<!-- measurements:end -->
## Эксперименты и отказы
Docker-стенд: Windows 10, Docker Desktop (VM 16 CPU, 15.2 GiB), клиент тестов — Python 3.11 на том же хосте, 2 реплики API по 1 CPU / 512 MiB. Числа нативной песочницы (2 vCPU на всё) — в `evidence/native-sandbox/summary.md`, с этими не сравнивать.
**Argon2id — выбор параметров** (`argon2_bench.py`, внутри контейнера api, медиана из 7):
| m, KiB | t | p | hash, мс | verify, мс | входов/с на реплику (2 параллельно) | RAM на хеширование |
| --- | --- | --- | --- | --- | --- | --- |
<!-- argon2:begin -->
| 19456 | 2 | 1 | 12.2 | 12.2 | 163.9 | 38 MiB |
| 19456 | 3 | 1 | 18.3 | 18.5 | 108.1 | 38 MiB |
| 32768 **(выбрано)** | 2 | 1 | 22.6 | 22.2 | 90.1 | 64 MiB |
| 47104 | 2 | 1 | 34.2 | 34.8 | 57.5 | 92 MiB |
| 65536 | 2 | 1 | 49.8 | 50.4 | 39.7 | 128 MiB |
| 65536 | 3 | 1 | 75.3 | 75.2 | 26.6 | 128 MiB |
<!-- argon2:end -->
Выбрано **m=32 MiB, t=2, p=1**: в 1,7 раза больше памяти, чем минимум OWASP/README (19 MiB, t=2), ~22 мс на проверку в контейнере api — незаметно для пользователя и вдвое дешевле 64 MiB (~50 мс). Бюджет реплики: не больше `ARGON2_MAX_CONCURRENCY=2` хешей одновременно (≤ 64 MiB из 512 MiB лимита памяти), ожидание слота не дольше 2 с, дальше — 503 + Retry-After. У api лимит 1 CPU, поэтому два параллельных хеша делят одно ядро: реальный потолок ≈ 1000 / 22 мс ≈ 45 проверок пароля в секунду на реплику (≈ 90/с на две; колонка «2 параллельно» — без учёта лимита CPU); всё, что выше, отсекают сначала общий лимит Redis (429), потом бюджет реплики (503), а не рост памяти или очередь. 64 MiB/t=2 удвоили бы и цену входа, и память на хеширование при том же бюджете, то есть вдвое снизили бы потолок входов. (В песочнице те же параметры стоили ~50 мс — выбор от этого не меняется.)
**Находка при проверке burst (песочница):** без настройки аллокатора glibc держит 32-MiB буферы Argon2 в отдельных arena каждого потока — 200 одновременных проверок раздували RSS реплики на +548 MiB (больше лимита 512 MiB → OOM-kill под атакой). `MALLOC_MMAP_THRESHOLD_=1048576` в окружении api заставляет крупные буферы идти через mmap и возвращаться ОС сразу: прирост RSS на том же burst — +37 MiB в песочнице, +74.7 MiB в Docker (ниже). Добавлено в `compose.yaml` и в угрозу T5.
**Login DoS** (`login_dos.py`):
<!-- dos:begin -->
| Фаза | Легитимных запросов | p50 / p95 / p99, мс | Коды легитимных | Попыток входа атаки | Коды атаки |
| --- | --- | --- | --- | --- | --- |
| без атаки, 20 с | 694 | 12.3 / 34.6 / 35.5 | {'200': 347, '201': 347} | — | — |
| атака: 16 потоков неверных паролей по 40 username, 20 с | 186 | 91.1 / 194.1 / 243.6 | {'200': 93, '201': 93} | 1924 (96.2/с), p95 ответа 480.5 мс | {'401': 360, '429': 1564} |
Из 1924 попыток атаки хеш Argon2id посчитан только для 360 (ответ 401: по 9 на каждый из 40 username до срабатывания лимита), 1564 отсечены общим (Redis) лимитом 429 + Retry-After до хеширования, 503 busy — 0: до бюджета реплики дело не дошло, его отдельно проверяет burst ниже. Легитимные запросы обслуживались без ошибок; задержка выросла (p95 34.6 → 194.1 мс), и поэтому closed-loop клиент сделал за 20 с меньше запросов (694 → 186): атакующий клиент, стенд и легитимный клиент делят одну машину, а 16 потоков TLS-рукопожатий и Python-клиента сами занимают CPU хоста.
Burst в одном процессе api (бюджет без помощи rate limit): 200 одновременных проверок пароля → одновременно считалось не больше 2 хешей (лимит 2), посчитано 75, отклонено 503 + Retry-After 125, самое долгое ожидание отказа 2.05 с (= очередь 2 с); прирост RSS 74.7 MiB при лимите 512 MiB (с MALLOC_MMAP_THRESHOLD_=1048576; без него в песочнице было +548 MiB, см. угрозу T5).
<!-- dos:end -->
**Жизненный цикл под нагрузкой** (`lifecycle.py`, фоновая нагрузка: вход + GET /users/me + POST message):
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
<!-- lifecycle:begin -->
| Ротация пароля Redis-пользователя api через OpenBao | 0 ошибок; после — активен только новый пароль | KV v5 → v6 (предыдущие версии — от прошлых прогонов); паролей во время перехода 2, после — 1 (новый); обе реплики подхватили за 23.9 с (≤ SECRETS_REFRESH_SECONDS 30 с); нагрузка 37.6 с, 880 ответов, ошибок 0 | без рестартов и пересборки; потерь данных нет |
| Ротация leaf-сертификата LB + отзыв старого (CRL) | 0 ошибок; старый leaf отвергается на новом соединении, открытые закрыты | старый leaf → connection_closed_after_handshake, новый → http_200; события API: {'tls.crl_loaded': 2, 'tls.revoked_connections_closed': 2, 'tls.client_rejected': 4}; нагрузка 23.1 с, 497 ответов, ошибок 0 | эффект отзыва ≤ CRL_RELOAD_SECONDS (5 с) после reload; потерь данных нет |
| Токены OpenBao роли api: TTL 20 с / max 40 с + отзыв всех токенов | сервисы перелогиниваются сертификатом, 0 ошибок | события API за 100 с: {'secrets.refresh_failed': 2, 'secrets.login': 4}; нагрузка 101.6 с, 2393 ответов, ошибок 0 | без участия оператора; потерь данных нет |
| OpenBao запечатан, одна реплика API перезапущена | работающая реплика обслуживает, перезапущенная не ready до unseal | sealed 75.6 с; перезапущенная реплика: starting (не healthy, LB её не использует); события API: {'secrets.refresh_failed': 5, 'secrets.waiting': 8, 'secrets.login': 1}; нагрузка 89.2 с, 2067 ответов, ошибок 0 | реплика ready через 5.4 с после unseal; потерь данных нет |
<!-- lifecycle:end -->
**Отказы lab3–5 через HTTPS** (публичные тесты): `run_persistence.py` — рестарт PostgreSQL, Redis, S3, RabbitMQ, **OpenBao** (+ unseal hook'ом), телеметрии, приложения и LB с сохранением volumes; данные и прежний токен на месте. `run_failover.py` — остановка каждой реплики по очереди под записью.
<!-- resilience:begin -->
- persistence: PASS — the hook restarted postgres, redis, s3, rabbitmq, openbao, api, janitor, dispatcher, worker, lb, otel-gate, otel-collector, prometheus, loki, tempo, grafana (all api replicas; same containers, new start time); data, the attachment bytes and the old token survived; hook + готовность 82.1 с
- failover, реплика 1: реплик до отказа 2, успешных записей 126, неудачных попыток (повторены тем же ключом) 0, первая запись после hook через 5.453 с
- failover, реплика 2: реплик до отказа 2, успешных записей 125, неудачных попыток (повторены тем же ключом) 0, первая запись после hook через 5.313 с
<!-- resilience:end -->
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Один хост, одна копия каждого хранилища** — отказ хоста, PostgreSQL, Redis, RabbitMQ, SeaweedFS или OpenBao не выдерживается (как в lab3–6). OpenBao sealed/недоступен: работающие реплики живут на кэше, новые не стартуют — сознательный fail closed.
- **Доли unseal в одном файле оператора** (учебный компромисс): restart-hook распечатывает OpenBao без человека. В эксплуатации — `bao.py init --print-shares-once`, доли у разных людей, `unseal --prompt`, отзыв root token после настройки, CA офлайн.
- **CRL проверяют** API (сам, с перечитыванием), HAProxy (lb, otel-gate — при reload), OpenBao cert auth (`bao.py sync-crl`). **Не проверяют**: RabbitMQ, Redis, PostgreSQL, Prometheus/Loki/Tempo — для них отзыв = удаление пользователя/смена пароля + короткий срок leaf (30 суток). Следующий шаг — короткоживущие leaf из OpenBao PKI с автопродлением (трек 5.2).
- **Loki/Tempo** различают только «наш CA / не наш»: читатель и писатель не разделены; сервис приложения может подписать телеметрию чужим `service.name` (шлюз проверяет identity, но не содержимое).
- **Пароли ролей PostgreSQL**: ротация описана (новая версия KV → `docker compose run --rm migrate`), но не демонстрировалась и даёт окно до 30 с отказов новых соединений; на 4 продемонстрирована ротация пароля Redis (без окна — два пароля одновременно).
- **SeaweedFS 4.52 после рестарта** отдаёт чтение раньше записи: `/cluster/healthz` = 200 через ~20 с, но мастер ещё ~15 с отвечает «master is warming up» на `/dir/assign`, и загрузки получают 503. Флага, сокращающего прогрев, нет; healthcheck S3 переведён на `/dir/assign` (готовность к записи), `test_consistency` ждёт первую успешную загрузку (в Docker — 14.7 с после того, как старый файл снова читается; в песочнице было 16 с).
- **Исключения hardening**: RabbitMQ с записываемым корнем; telemetry-init — root + CAP_CHOWN на время chown (идемпотентно: chown только для volume, который ещё не принадлежит 10001). SeaweedFS после Docker-прогона работает от uid 1000 (`seaweed` образа), исключение uid 0 снято.
- **Restart hook на настоящем Docker** (в песочнице не воспроизводилось): (1) `docker compose restart` восьми хранилищ ждёт самые медленные (PostgreSQL, RabbitMQ), а запечатанный OpenBao за это время проваливает 3 healthcheck, становится `unhealthy`, и `up --wait` сразу падает, хотя через секунду OpenBao уже распечатан; (2) параллельный `restart otel-gate otel-collector …` — Collector не может войти в network namespace ещё остановленного gate. Исправлено порядком в `scripts/restart.cmd`/`.sh`: OpenBao перезапускается отдельно и распечатывается сразу (внутри `start_period`), otel-gate — отдельно перед Collector; набор перезапускаемых сервисов тот же.
- **Нестабильные проверки, найденные в Docker** (ожидания не менялись, исправлено ожидание готовности): Loki после старта ~15 с отвечает `/ready` → 503 «Ingester not ready» — положительная проверка mTLS оператора к Loki/Tempo в `tls_probe.py` ждёт готовности до 30 с, только если mTLS пройден (ответ 503), итог по-прежнему должен быть 200; в `test_telemetry.py` трасса считалась полной по span'ам worker, а batch API/dispatcher выгружается независимо (до 5 с) и иногда приходил в Tempo позже (2 FAIL из 4) — условие ожидания теперь требует span'ы всех процессов, проверки те же (4 из 4 PASS). `lifecycle.py token-expiry`: после `revoke-service-tokens` отзыв замечает то обращение к OpenBao, которое случится раньше, — чтение KV (`secrets.refresh_failed`) или продление токена; во втором случае API сразу входил заново, но писал общее с max TTL событие `secrets.relogin`, и тест (ждал только refresh_failed) падал при работающей функции. Теперь неудачный renew — отдельное событие `secrets.renew_failed` (WARNING, `secretstore.py`), тест принимает любой из двух путей обнаружения и по-прежнему требует повторный вход и 0 ошибок. В `upload_memory.py` счётчик загрузок по репликам сравнивал X-Instance-Id с ID контейнера (в lab7 это alias) и всегда давал 0 — сопоставление через `instance_aliases()`.
- **Память SeaweedFS 4.52**: ~240 MiB в простое; в Docker-прогоне `upload_memory.py` (20 параллельных загрузок по ~10 MiB) довёл процесс до лимита 512 MiB, ядро убило его (SIGKILL, рестарт по политике), половина загрузок получила 503. Исправлено без повышения лимита: `GOMEMLIMIT=400MiB` (GC Go ориентируется на лимит) и `-s3.concurrentUploadLimitMB=128` (сверх 128 MiB тел одновременно шлюз ставит загрузки в очередь). После — все 20 загрузок 202, рестартов S3 нет. Цена — задержка загрузок при длительном потоке крупных файлов (T14).
- **`localhost` на хосте Windows** резолвится сначала в `::1`, а порты опубликованы только на `127.0.0.1` (IPv6-loopback не публикуем намеренно): каждое новое TCP-соединение с `https://localhost:8443` ждёт ~2 с отказа по IPv6 (замер: localhost 2031–2051 мс, 127.0.0.1 — 0 мс). Первый Docker-прогон из-за этого давал в 40 раз меньше запросов нагрузки. Скрипты `student-tests` по умолчанию ходят на `https://127.0.0.1:<порт>` (IP есть в SAN сертификатов ingress и админ-интерфейсов, имя проверяется так же строго); `https://localhost:8443` работает, только медленнее на новых соединениях.
- **OpenBao file storage** в 2.6.2 помечен deprecated (в логе: «use bao operator migrate … by v2.7.0»). Для учебного стенда на одном хосте оставлен; перед обновлением до 2.7 — `bao operator migrate` на integrated storage (raft).
- **Rate limit на username** позволяет заблокировать вход конкретному пользователю на окно (60 с) — плата за защиту от подбора; лимит регистрации при недоступном Redis пропускается (защищает бюджет Argon2id).
- **Docker-прогон после песочницы** (настоящие OpenBao 2.6.2, HAProxy 3.2, RabbitMQ 4.1.4, PostgreSQL 17, Redis 7.4, Loki/Tempo/Grafana с TLS): прошли все 46 шагов. Исправлено по его итогам (подробно — пункты выше): SeaweedFS 4.52 не мог писать в `/data` от root без capabilities → запуск от uid 1000; `telemetry-init` падал на повторном `up` → идемпотентный chown; у Prometheus не было tmpfs `/tmp` для `promtool test rules`; память SeaweedFS под 20 параллельными загрузками; порядок restart hook; `localhost` → `127.0.0.1` в клиентах тестов; ожидания готовности в трёх тестах; отдельное событие `secrets.renew_failed`; по скану образов — `apt-get upgrade` в `Dockerfile` (OpenSSL 3.5.1 → 3.5.7), после чего весь прогон повторён.
- **Открыто по скану** (`security/scan-triage.md`): OpenSSL 3.5.4 в `rabbitmq:4.1.4-alpine` (порт только в internal-сети `data`, вход по mTLS), Go stdlib/x/net в OpenBao 2.6.2, musl/pcre2/zlib в HAProxy 3.2.25 — закрываются обновлением тегов до патч-релизов с повторным прогоном; в этой работе версии сторонних сервисов не менялись. Конфигурации OpenBao, HAProxy, RabbitMQ (EXTERNAL), PostgreSQL (ключ в /tmp/pgtls), TLS Collector → Loki/Tempo/Prometheus, Grafana → источники и внутренний gRPC Tempo заработали без изменений.
## Ответы на вопросы защиты (кратко)
- **Какая часть mTLS проверяет шифрование, какая идентичность и где разрешение на действие?** Шифрование и целостность — TLS-сессия (ECDHE). Идентичность сервера — проверка цепочки до нашего CA и имени (verify-full / check_hostname / verifyhost). Идентичность клиента — его сертификат (цепочка, срок, CRL). Разрешение — отдельный шаг после рукопожатия: API — allowlist CN=lb; otel-gate — CN ∈ приложений; RabbitMQ — пользователь = CN и его permissions; OpenBao — роль cert auth = CN → policy; Prometheus — `client_allowed_sans`; PostgreSQL/Redis/S3 — по роли/ACL/ключу. Пользовательская сессия (Bearer) — ещё выше, внутри API.
- **Почему доверенный CA ещё не означает, что любой его клиент может быть LB?** Один CA выпускает сертификаты всем сервисам; «подписан нашим CA» говорит только «это кто-то из наших». Если пускать любого, скомпрометированный worker стал бы балансировщиком (обход лимитов, подделка XFF). Поэтому API проверяет CN=lb — в тесте сертификаты worker, api и intruder нашего CA отвергаются.
- **Где хранится первый секрет для доступа к OpenBao и кто имеет право unseal?** Первый секрет сервиса — его leaf-сертификат и ключ (`secrets/pki/<identity>`, смонтирован только в этот сервис, 30 суток, отзывается CRL). SecretID/токенов в env нет. Unseal — оператор: 3 доли, порог 2 (на стенде в `secrets/operator/openbao-init.json`, 0600, вне Git и контейнеров; в эксплуатации — у разных людей).
- **Что происходит при истечении credentials, остановке OpenBao и рестарте API?** Токен OpenBao продлевается и при исчерпании max TTL заменяется новым входом по сертификату (без влияния на запросы). Истёкший leaf — отказ рукопожатия, нужен `pki.py rotate`. OpenBao остановлен/sealed — работающие процессы обслуживают запросы с секретами в памяти, обновление ротаций откладывается; рестартованная реплика не становится ready, LB её не использует; после unseal — восстанавливается сама. Рестарт API при работающем OpenBao — обычный: вход сертификатом, секреты за доли секунды.
+80
View File
@@ -0,0 +1,80 @@
"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202.
Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого
объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение:
1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503.
2. PUT исходника в staging/<hex> — упало: удаляем объект (PUT мог пройти)
и строку, клиенту 503. Если и это не вышло —
uploading-строку уберёт janitor после grace.
3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT
— упало/неизвестно: клиенту 503, строка
остаётся uploading -> janitor удалит объект
и строку. Если COMMIT всё же прошёл, задание
есть и будет обработано (повтор клиента даст
второе вложение — в контракте нет ключа).
4. 202 Attachment(status=queued) — только после шага 3.
Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный
брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно.
"""
import logging
from uuid import UUID, uuid4
import db
import settings
import storage
from errors import unavailable
from uploads import ImageInfo, Upload
log = logging.getLogger("messenger.attachments")
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
attachment_id = uuid4()
key = storage.new_key(uuid4().hex)
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
upload.size, upload.sha256, image.width, image.height, key)
try:
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
except storage.StorageUnavailable:
compensate(attachment_id, key)
raise unavailable("File storage is temporarily unavailable") from None
row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID)
if row is None:
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
log.warning("Upload %s was reclaimed before it was queued", attachment_id)
raise unavailable()
log.info("Attachment %s accepted into staging and queued", attachment_id)
return {**row, "variants": []}
def compensate(attachment_id: UUID, key: str) -> None:
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
try:
storage.delete(key, quick=True)
db.delete_upload_intent(attachment_id)
log.info("Upload %s rolled back after S3 failure", attachment_id)
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
def public(row: dict) -> dict:
"""Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей)."""
return {
"id": row["id"],
"chat_id": row["chat_id"],
"owner_id": row["owner_id"],
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["state"],
"variants": [
{key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
for v in row.get("variants", [])
],
"error_code": row["error_code"] if row["state"] == "failed" else None,
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
+268
View File
@@ -0,0 +1,268 @@
"""Пароли (Argon2id), сессии и ограничение частоты входа (lab7).
Пароли: argon2-cffi, Argon2id, параметры из settings (не ниже m=19 MiB, t=2, p=1; выбор — по замеру,
REPORT.md). В БД хранится строка $argon2id$v=19$m=...,t=...,p=...$<salt>$<hash>: соль и параметры
вместе с хешем. При успешном входе hash со старыми параметрами пересчитывается (rehash) и сохраняется.
Бюджет CPU/RAM: Argon2id намеренно дорог, поэтому поток запросов входа/регистрации — готовый DoS.
- одновременно в реплике не больше ARGON2_MAX_CONCURRENCY хешей (память N x m, CPU N ядер);
ожидание слота не дольше ARGON2_QUEUE_SECONDS, иначе 503 + Retry-After (нет растущей очереди);
- ДО хеширования — общий для всех реплик счётчик в Redis (фиксированное окно): попыток на username
и на IP клиента; превышение -> 429 + Retry-After, хеш не считается вовсе.
- несуществующий пользователь проверяется против DUMMY_HASH с теми же параметрами: время ответа
не выдаёт, есть ли такой username.
Сессии: случайный токен 256 бит; в Redis хранится только SHA-256 токена (session:<hex>) -> user_id
с абсолютным временем истечения. Redis — TLS + клиентский сертификат + ACL-пользователь api, у которого
есть доступ только к ключам session:* и ratelimit:* и только к нужным командам; пароль ACL-пользователя
берётся из OpenBao на каждое новое соединение (ротация без рестарта).
"""
import base64
import binascii
import hashlib
import logging
import secrets
import ssl
import threading
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from uuid import UUID
import redis
from argon2 import PasswordHasher, Type
from argon2.exceptions import InvalidHashError, VerificationError
from redis.backoff import NoBackoff
from redis.credentials import CredentialProvider
from redis.retry import Retry
import settings
import telemetry
from errors import BASIC_CHALLENGE, busy, rate_limited, unauthorized, unavailable
from secretstore import store
log = logging.getLogger("messenger.auth")
hasher = PasswordHasher(time_cost=settings.ARGON2_ITERATIONS, memory_cost=settings.ARGON2_MEMORY_KIB,
parallelism=settings.ARGON2_PARALLELISM, hash_len=32, salt_len=16, type=Type.ID)
DUMMY_HASH = hasher.hash("there-is-no-such-user")
PASSWORD_MIN_LENGTH = 12
PASSWORD_MAX_LENGTH = 128
_hash_slots = threading.BoundedSemaphore(settings.ARGON2_MAX_CONCURRENCY)
HASH_REJECTED = telemetry.meter.create_counter(
"messenger.auth.hash_rejected", unit="{request}",
description="Password hashing requests rejected because the per-replica Argon2id budget was exhausted")
RATE_LIMITED = telemetry.meter.create_counter(
"messenger.auth.rate_limited", unit="{request}", description="Login/registration attempts answered with 429")
REHASHED = telemetry.meter.create_counter(
"messenger.auth.rehashed", unit="{user}", description="Password hashes upgraded to current Argon2id parameters")
class _HashSlot:
"""Слот бюджета хеширования; занят — ждём не дольше ARGON2_QUEUE_SECONDS, иначе 503."""
def __enter__(self):
if not _hash_slots.acquire(timeout=settings.ARGON2_QUEUE_SECONDS):
HASH_REJECTED.add(1)
log.warning("Argon2id budget exhausted", extra={"event": "auth.hash_budget_exhausted"})
raise busy(1)
return self
def __exit__(self, *exc):
_hash_slots.release()
return False
def hash_password(password: str) -> str:
# Argon2id нагружает CPU обработчика (а не БД): в трассе это отдельный спан.
with _HashSlot(), telemetry.internal_span("argon2id hash"):
return hasher.hash(password)
def password_matches(password_hash: str | None, password: str) -> bool:
with _HashSlot(), telemetry.internal_span("argon2id verify"):
try:
hasher.verify(password_hash or DUMMY_HASH, password)
except (VerificationError, InvalidHashError):
return False
return password_hash is not None
def needs_rehash(password_hash: str) -> bool:
try:
return hasher.check_needs_rehash(password_hash)
except InvalidHashError:
return False
def rehash(password: str) -> str:
return hash_password(password)
def hash_parameters(password_hash: str) -> dict:
"""Параметры из строки хеша (для тестов/отчёта): {"type": "argon2id", "m": ..., "t": ..., "p": ...}."""
parts = password_hash.split("$")
params = dict(item.split("=") for item in parts[3].split(","))
return {"type": parts[1], "m": int(params["m"]), "t": int(params["t"]), "p": int(params["p"]),
"salt_bytes": len(base64.b64decode(parts[4] + "=" * (-len(parts[4]) % 4)))}
def parse_basic(header: str | None) -> tuple[str, str] | None:
"""'Basic base64(username:password)' -> (username, password) или None.
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
"""
scheme, _, value = (header or "").strip().partition(" ")
if scheme.lower() != "basic" or not value or len(value) > 1024:
return None
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError):
return None
username, colon, password = decoded.partition(":")
if not colon or not username or not username.isascii():
return None
return username, password
def parse_bearer(header: str | None) -> str | None:
scheme, _, value = (header or "").strip().partition(" ")
value = value.strip()
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
return None
return value
KEY_PREFIX = "session:"
RATE_PREFIX = "ratelimit:"
# Атрибуты спанов Redis: система и операция. Ключ (session:<SHA-256 токена>) и значения не пишутся.
REDIS_ATTRS = {"db.system.name": "redis", "server.address": "redis"}
class _RedisCredentials(CredentialProvider):
"""ACL-пользователь и его пароль — текущая версия из OpenBao на каждое новое соединение."""
def get_credentials(self):
return settings.REDIS_USER, store.get("redis_password")
_redis = redis.Redis(
connection_pool=redis.BlockingConnectionPool(
connection_class=redis.SSLConnection,
host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=0,
credential_provider=_RedisCredentials(),
ssl_ca_certs=settings.TLS_CA_FILE,
ssl_certfile=settings.TLS_CERT_FILE,
ssl_keyfile=settings.TLS_KEY_FILE,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_check_hostname=True,
ssl_min_version=ssl.TLSVersion.TLSv1_2,
max_connections=settings.REDIS_POOL_MAX,
timeout=settings.REDIS_TIMEOUT_MS / 1000,
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
lib_name=None, # без CLIENT SETINFO: ACL-пользователю api эта команда не нужна и не выдана
lib_version=None,
)
)
@dataclass(frozen=True)
class Session:
user_id: UUID
key: str
def session_key(token: str) -> str:
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
def _redis_failed(exc: Exception, where: str):
log.warning("Redis is unavailable (%s): %s", where, type(exc).__name__,
extra={"event": "redis.unavailable", "error.type": type(exc).__name__})
return unavailable()
def check_rate(kind: str, subject: str, limit: int, fail_open: bool = False) -> None:
"""Общий для реплик счётчик попыток (фиксированное окно). Превышение -> 429 + Retry-After.
В ключе — SHA-256 субъекта (username/IP), а не сам субъект: Redis не хранит имена и адреса.
fail_open: при недоступном Redis пропустить (регистрация с lab2 работает без Redis; от перегрузки
её по-прежнему защищает бюджет Argon2id реплики). Вход без Redis невозможен и так (сессии в Redis).
"""
key = RATE_PREFIX + kind + ":" + hashlib.sha256(subject.encode("utf-8")).hexdigest()[:32]
window_ms = settings.LOGIN_RATE_WINDOW_SECONDS * 1000
try:
with telemetry.client_span("redis INCR", REDIS_ATTRS | {"db.operation.name": "INCR"}):
pipe = _redis.pipeline(transaction=True)
pipe.incr(key)
pipe.pexpire(key, window_ms, nx=True)
pipe.pttl(key)
count, _, ttl_ms = pipe.execute()
except redis.RedisError as exc:
if fail_open:
log.warning("Rate limit skipped, Redis is unavailable: %s", type(exc).__name__,
extra={"event": "auth.rate_limit_skipped", "error.type": type(exc).__name__})
return
raise _redis_failed(exc, "rate limit") from None
if count > limit:
RATE_LIMITED.add(1, {"messenger.rate_limit": kind})
log.info("Rate limited: %s", kind, extra={"event": "auth.rate_limited", "messenger.rate_limit": kind})
raise rate_limited(-(-max(ttl_ms, 1000) // 1000))
def create_session(user_id: UUID) -> tuple[str, datetime]:
"""Новая сессия: случайный токен и абсолютное время истечения."""
token = secrets.token_urlsafe(32)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
try:
with telemetry.client_span("redis SET", REDIS_ATTRS | {"db.operation.name": "SET"}):
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
except redis.RedisError as exc:
raise _redis_failed(exc, "login") from None
return token, expires_at
def check_bearer(header: str | None) -> Session:
token = parse_bearer(header)
if token is None:
raise unauthorized("Bearer token is required")
key = session_key(token)
try:
with telemetry.client_span("redis GET", REDIS_ATTRS | {"db.operation.name": "GET"}):
user_id = _redis.get(key)
except redis.RedisError as exc:
raise _redis_failed(exc, "session") from None
if user_id is None:
raise unauthorized("Session is invalid or expired")
return Session(UUID(user_id), key)
def delete_session(session: Session) -> None:
try:
with telemetry.client_span("redis DEL", REDIS_ATTRS | {"db.operation.name": "DEL"}):
_redis.delete(session.key)
except redis.RedisError as exc:
raise _redis_failed(exc, "logout") from None
def ping_redis() -> None:
try:
_redis.ping()
except redis.RedisError:
raise unavailable() from None
def login_failed():
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
def close() -> None:
_redis.close()
_redis.connection_pool.disconnect()
+133
View File
@@ -0,0 +1,133 @@
"""RabbitMQ: подключение (lab7: AMQPS + mTLS, EXTERNAL), проверка топологии, публикация с publisher confirms.
Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует
scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure:
они только проверяют её пассивным declare и не могут создать/удалить очередь.
exchange attachments (direct, durable)
--attachment.process.v1--> queue attachments.process.v1 (quorum, durable)
x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once),
x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря)
exchange attachments.dlx (direct, durable)
--attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py
через basic_get + nack не расходует delivery-limit)
Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до
подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются.
"""
import json
import logging
import pika
from pika.exceptions import AMQPError
import pipeline
import settings
import tlsutil
log = logging.getLogger("messenger.broker")
logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал
BrokerError = (AMQPError, OSError)
def parameters(url: str | None = None) -> pika.URLParameters:
"""lab7: amqps://rabbitmq:5671/messenger — TLS с проверкой CA и имени, клиентский сертификат сервиса,
вход механизмом EXTERNAL (пользователь брокера = CN сертификата: worker / dispatcher). Пароля нет."""
params = pika.URLParameters(url or settings.BROKER_URL)
host = params.host
params.ssl_options = pika.SSLOptions(tlsutil.client_context(), server_hostname=host)
params.credentials = pika.credentials.ExternalCredentials()
params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS
params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно
params.socket_timeout = 5
params.connection_attempts = 1
params.client_properties = {"connection_name": settings.INSTANCE_ID}
return params
def connect(url: str | None = None) -> pika.BlockingConnection:
return pika.BlockingConnection(parameters(url))
def check_topology(channel, need=("exchange", "queue", "dlq")) -> dict:
"""Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях.
lab7: каждая проверка — на отдельном временном канале. RabbitMQ 4.2+ отвечает 403 на пассивный declare
ресурса, на который у пользователя нет НИКАКИХ прав (worker не видит DLQ и exchange, dispatcher — основную
очередь): такой ресурс помечается None, а рабочий канал вызывающего не закрывается. Прав не расширяем.
Если не виден ни один ресурс — это ошибка (нет топологии/прав вообще).
"""
connection = channel.connection
def passive(declare):
temp = connection.channel()
try:
return declare(temp)
except pika.exceptions.ChannelClosedByBroker as exc:
if exc.reply_code == 403:
return None
raise
finally:
if temp.is_open:
temp.close()
exchange = passive(lambda ch: ch.exchange_declare(pipeline.EXCHANGE, passive=True)) if "exchange" in need else None
main = passive(lambda ch: ch.queue_declare(pipeline.QUEUE, passive=True).method) if "queue" in need else None
dlq = passive(lambda ch: ch.queue_declare(pipeline.DLQ, passive=True).method) if "dlq" in need else None
if all(item is None for item in (exchange, main, dlq)):
raise pika.exceptions.ChannelClosedByBroker(403, "no broker resource is visible to this identity")
return {"queue": main.message_count if main else None, "consumers": main.consumer_count if main else None,
"dlq": dlq.message_count if dlq else None}
def properties(message: dict) -> pika.BasicProperties:
# W3C Trace Context в native headers AMQP (основной carrier для worker) и в теле (поле схемы).
headers = {key: message[key] for key in ("traceparent", "tracestate") if key in message}
return pika.BasicProperties(
delivery_mode=pika.DeliveryMode.Persistent,
content_type="application/json",
message_id=f"{message['job_id']}:{message['attempt']}",
type=message["type"],
headers=headers or None,
)
class Publisher:
"""Канал в режиме confirm: basic_publish возвращает управление только после ack брокера.
mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError.
Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях
строка outbox остаётся неопубликованной и будет отправлена позже.
"""
def __init__(self):
self.connection = None
self.channel = None
def open(self):
if self.channel is None or self.channel.is_closed or self.connection.is_closed:
self.close()
self.connection = connect()
self.channel = self.connection.channel()
self.channel.confirm_delivery()
check_topology(self.channel)
log.info("connected to broker, publisher confirms enabled", extra={"event": "broker.connected"})
return self.channel
def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY):
body = json.dumps(message, separators=(",", ":")).encode("utf-8")
self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True)
def pump(self):
if self.connection is not None and self.connection.is_open:
self.connection.process_data_events(time_limit=0) # heartbeat в паузах
def close(self):
try:
if self.connection is not None and self.connection.is_open:
self.connection.close()
except BrokerError:
pass
self.connection = None
self.channel = None
+731
View File
@@ -0,0 +1,731 @@
name: messenger-lab7
# Lab7: безопасность. Порядок первого запуска (REPORT.md, «Запуск из чистого clone»):
# pki.py init -> init_secrets.py -> up openbao -> bao.py init/unseal/configure -> up --build --wait
# В репозитории нет ни одного секрета: secrets/ (CA, ключи, пароли) создаёт оператор локально, .gitignore.
#
# Сети (internal = без выхода наружу и без публикации портов):
# ingress — только lb; публикуется 127.0.0.1:8443 (HTTPS) и 127.0.0.1:8404 (статистика, mTLS оператора)
# frontend — internal: lb <-> api (mTLS, у API только CN=lb)
# data — internal: приложение <-> PostgreSQL / Redis / S3 / RabbitMQ (все TLS)
# secrets — internal: сервисы <-> OpenBao (TLS + cert auth)
# telemetry — internal: приложение -> otel-gate (mTLS) -> Collector -> Prometheus/Loki/Tempo (mTLS) <- Grafana
# ops — только для публикации админ-интерфейсов на 127.0.0.1 (OpenBao, Prometheus, Loki, Tempo, Grafana)
x-app-build: &app-build
context: .
args:
APP_VERSION: ${APP_VERSION:-dev}
# Общие меры для контейнеров: только чтение корня, без capabilities, без повышения прав, не root
# (USER 10001 в Dockerfile приложения; у остальных — user образа). Пишут только в tmpfs /tmp и свои volumes.
# Число процессов ограничено в deploy.resources.limits.pids каждого сервиса.
x-hardening: &hardening
read_only: true
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
# TLS-файлы процессов приложения (смонтированы каталогами: ротация leaf видна без пересоздания контейнера)
# и адрес OpenBao. BAO_ROLE = CN сертификата сервиса = роль cert auth = policy.
x-tls-env: &tls-env
TLS_CA_FILE: /run/tls-ca/ca.crt
TLS_CRL_FILE: /run/tls-ca/crl.pem
TLS_CERT_FILE: /run/tls/tls.crt
TLS_KEY_FILE: /run/tls/tls.key
BAO_ADDR: https://openbao:8200
SECRETS_REFRESH_SECONDS: ${SECRETS_REFRESH_SECONDS:-30}
x-db-env: &db-env
DB_HOST: postgres
DB_PORT: "5432"
DB_NAME: messenger
DB_SSLMODE: verify-full
x-pipeline-env: &pipeline-env
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
# Учебный стенд: точки внедрения сбоев (срабатывают только по файлу в volume faults worker/dispatcher).
FAULT_INJECTION: ${FAULT_INJECTION:-1}
FAULTS_DIR: /faults
# lab6 -> lab7: OTLP/HTTP по mTLS к otel-gate (имя otel-collector), сертификат сервиса = его identity.
x-otel-env: &otel-env
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls-ca/ca.crt
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/tls.crt
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/tls.key
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_METRIC_EXPORT_INTERVAL: "10000"
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
x-otel-depends: &otel-depends
condition: service_started
required: false
x-alive-check: &alive-check
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
interval: 10s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
x-bao-ready: &bao-ready
condition: service_healthy
services:
# ---------------- секреты ----------------
# OpenBao 2.6.2, НЕ dev-mode: file storage в volume, TLS-listener, после старта sealed до unseal оператором.
# healthy = unsealed (bao status: 0 — unsealed, 2 — sealed). 127.0.0.1:8200 — только для оператора.
openbao:
image: openbao/openbao:2.6.2
command: ["server"] # entrypoint образа: bao server -config=/openbao/config
user: "100" # пользователь openbao образа, не root
environment:
SKIP_SETCAP: "true"
SKIP_CHOWN: "true"
BAO_ADDR: https://127.0.0.1:8200
BAO_CACERT: /run/tls-ca/ca.crt
volumes:
- ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro
- ./secrets/pki/openbao:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- openbaodata:/openbao/file
ports:
- "127.0.0.1:8200:8200"
networks: [secrets, ops]
<<: *hardening
tmpfs: ["/tmp:size=16m,mode=1777"]
healthcheck:
test: ["CMD", "bao", "status", "-address=https://127.0.0.1:8200", "-ca-cert=/run/tls-ca/ca.crt"]
interval: 5s
timeout: 3s
retries: 3
start_period: 5s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 256M, pids: 256}
# ---------------- хранилища ----------------
postgres:
image: postgres:17.11-alpine
user: "70:70" # postgres образа alpine; entrypoint не делает chown (не нужен CAP_CHOWN)
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
# Ключ TLS копируется в tmpfs с правами 0600 (PostgreSQL отказывается от ключа, доступного группе/всем,
# а права bind mount на Windows не управляются). Только hostssl + SCRAM (infra/postgres/pg_hba.conf).
entrypoint: ["sh", "-c", "install -d -m 0700 /tmp/pgtls && install -m 0600 /run/tls/tls.key /tmp/pgtls/tls.key && install -m 0644 /run/tls/tls.crt /tmp/pgtls/tls.crt && exec docker-entrypoint.sh \"$$@\"", "--"]
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on",
"-c", "ssl=on", "-c", "ssl_cert_file=/tmp/pgtls/tls.crt", "-c", "ssl_key_file=/tmp/pgtls/tls.key",
"-c", "ssl_min_protocol_version=TLSv1.2", "-c", "password_encryption=scram-sha-256",
"-c", "hba_file=/etc/postgresql/pg_hba.conf", "-c", "log_connections=on"]
secrets: [postgres_password]
volumes:
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
- ./secrets/pki/postgres:/run/tls:ro
- pgdata:/var/lib/postgresql/data
networks: [data]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:size=16m,mode=1777", "/var/run/postgresql:size=1m,uid=70,gid=70,mode=0775"]
shm_size: 128m
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] # unix-socket внутри контейнера
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
redis:
image: redis:7.4.11-alpine
user: "999:1000" # redis образа; без root и без chown
# Стартовый ACL копируется в volume один раз; дальше ACL живёт в volume (ACL SAVE после ротации).
command: ["sh", "-c", "[ -f /data/users.acl ] || cp /run/redis-acl/users.acl /data/users.acl; exec redis-server /usr/local/etc/redis/redis.conf"]
volumes:
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
- ./secrets/redis:/run/redis-acl:ro
- ./secrets/pki/redis:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- redisdata:/data
networks: [data]
<<: *hardening
tmpfs: ["/tmp:size=8m,mode=1777"]
healthcheck:
# Жив и отвечает по TLS; без AUTH сервер отвечает NOAUTH — это и есть «работает и закрыт».
test: ["CMD-SHELL", "redis-cli --tls --cacert /run/tls-ca/ca.crt --cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis ping 2>&1 | grep -qE 'NOAUTH|PONG'"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
deploy:
resources:
limits: {cpus: "0.50", memory: 128M, pids: 256}
# SeaweedFS 4.52: master/volume/filer слушают только 127.0.0.1 (-ip.bind), наружу — только S3 HTTPS :8333
# (-s3.ip.bind=0.0.0.0) и gRPC шлюза :18333 под mTLS CN=s3 (security.toml). Iceberg/Lance/IAM API выключены.
s3:
image: chrislusf/seaweedfs:4.52
entrypoint: ["/usr/bin/weed"]
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
"-master.volumeSizeLimitMB=1024", "-volume.max=0", "-master.telemetry=false",
"-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0",
"-s3.cert.file=/run/tls/tls.crt", "-s3.key.file=/run/tls/tls.key",
"-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.iam=false",
"-s3.concurrentUploadLimitMB=128",
"-s3.config=/run/secrets/s3_config"]
# Docker-прогон: ~240 MiB в простое, 20 параллельных загрузок по ~10 MiB доводили процесс до лимита 512 MiB
# и SIGKILL (cgroup OOM) — API получал 503. Go GC теперь ориентируется на лимит, а сверх 128 MiB тел
# загрузок одновременно S3-шлюз ставит запросы в очередь вместо роста памяти.
environment:
GOMEMLIMIT: 400MiB
secrets: [s3_config]
volumes:
- ./infra/seaweedfs/security.toml:/etc/seaweedfs/security.toml:ro
- ./secrets/pki/s3:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- s3data:/data
networks: [data]
# Пользователь seaweed образа 4.52 (uid/gid 1000, владелец /data). Entrypoint образа делает chown + su-exec
# от root; мы его не используем (entrypoint = weed), поэтому сразу не root и без capabilities.
user: "1000:1000"
<<: *hardening
tmpfs: ["/tmp:size=64m,mode=1777"]
healthcheck:
# Готовность к ЗАПИСИ: после рестарта мастер ~15 c отвечает "master is warming up" на
# /dir/assign, хотя /cluster/healthz уже 200. Проверяем, что мастер выдаёт fid.
test: ["CMD-SHELL", "wget -q -O - 'http://127.0.0.1:9333/dir/assign?count=1' | grep -q '\"fid\"'"]
interval: 5s
timeout: 3s
retries: 24
start_period: 15s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
rabbitmq:
image: rabbitmq:4.1.4-alpine
hostname: rabbitmq
user: rabbitmq
environment:
RABBITMQ_NODENAME: rabbit@localhost # distribution только на loopback (infra/rabbitmq.conf)
ERL_EPMD_ADDRESS: 127.0.0.1
volumes:
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
- ./infra/rabbitmq-enabled-plugins:/etc/rabbitmq/enabled_plugins:ro
- ./secrets/pki/rabbitmq:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- rabbitmqdata:/var/lib/rabbitmq
secrets: [rabbitmq_definitions]
networks: [data]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
healthcheck:
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
interval: 5s
timeout: 10s
retries: 24
start_period: 20s
start_interval: 2s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 512}
# ---------------- bootstrap ----------------
migrate:
build: *app-build
command: ["python", "migrate.py"]
environment:
<<: [*tls-env, *db-env]
BAO_ROLE: migrate
DB_USER: messenger
DB_PASSWORD_KEY: db_owner_password
volumes:
- ./secrets/pki/migrate:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
networks: [data, secrets]
depends_on:
postgres: {condition: service_healthy}
openbao: *bao-ready
<<: *hardening
tmpfs: ["/tmp:size=8m,mode=1777"]
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.50", memory: 128M, pids: 256}
s3-init:
build: *app-build
command: ["python", "s3_init.py"]
environment:
<<: *tls-env
BAO_ROLE: s3-init
S3_ENDPOINT: https://s3:8333
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
volumes:
- ./secrets/pki/s3-init:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
networks: [data, secrets]
depends_on:
s3: {condition: service_healthy}
openbao: *bao-ready
<<: *hardening
tmpfs: ["/tmp:size=8m,mode=1777"]
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 128M, pids: 256}
# ---------------- приложение ----------------
api:
build: *app-build
env_file: .env
environment:
<<: [*tls-env, *db-env, *otel-env]
APP_HOST: 0.0.0.0
APP_PORT: "8443"
API_TLS: "1"
API_ALLOWED_CLIENTS: lb # identity, которой разрешено подключаться к API (mTLS)
BAO_ROLE: api
DB_USER: messenger_api
REDIS_HOST: redis
REDIS_USER: api
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
S3_ENDPOINT: https://s3:8333
BROKER_URL: "" # у API нет доступа к брокеру: задание уходит через outbox
UPLOAD_TMP_DIR: /tmp/uploads
# Буферы Argon2id (32 MiB) — всегда через mmap и сразу назад ОС. Без этого glibc оставляет освобождённые
# буферы в per-thread arenas пула потоков Starlette: замер (login_dos.py, 200 проверок) — RSS +548 MiB
# против +37 MiB с этой настройкой; при лимите контейнера 512 MiB это разница между OOM и запасом.
MALLOC_MMAP_THRESHOLD_: "1048576"
OTEL_SERVICE_NAME: messenger-api
volumes:
- ./secrets/pki/api:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
networks: [frontend, data, secrets, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
redis: {condition: service_healthy}
openbao: *bao-ready
otel-gate: *otel-depends
<<: *hardening
# Временные файлы текущих загрузок (до 10 MiB каждый) — tmpfs, учитывается в лимите памяти контейнера.
tmpfs: ["/tmp:size=160m,mode=1777"]
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
janitor:
build: *app-build
command: ["python", "janitor.py"]
env_file: .env
environment:
<<: [*tls-env, *db-env, *otel-env]
BAO_ROLE: janitor
DB_USER: messenger_janitor
S3_ENDPOINT: https://s3:8333
BROKER_URL: ""
OTEL_SERVICE_NAME: messenger-janitor
volumes:
- ./secrets/pki/janitor:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
networks: [data, secrets, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
openbao: *bao-ready
otel-gate: *otel-depends
<<: *hardening
tmpfs: ["/tmp:size=16m,mode=1777"]
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 192M, pids: 256}
dispatcher:
build: *app-build
command: ["python", "dispatcher.py"]
environment:
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
BAO_ROLE: dispatcher
DB_USER: messenger_dispatcher
BROKER_URL: amqps://rabbitmq:5671/messenger # не секрет: вход по сертификату CN=dispatcher
OTEL_SERVICE_NAME: messenger-dispatcher
volumes:
- ./secrets/pki/dispatcher:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- faults:/faults
networks: [data, secrets, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
openbao: *bao-ready
otel-gate: *otel-depends
<<: *hardening
tmpfs: ["/tmp:size=16m,mode=1777"]
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 128M, pids: 256}
# Worker: обработка изображения в дочернем процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB,
# RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS; контейнер ограничен по CPU/RAM/PID.
worker:
build: *app-build
command: ["python", "worker.py"]
environment:
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
BAO_ROLE: worker
DB_USER: messenger_worker
BROKER_URL: amqps://rabbitmq:5671/messenger # вход по сертификату CN=worker
S3_ENDPOINT: https://s3:8333
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
WORKER_TMP_DIR: /tmp/work
OTEL_SERVICE_NAME: messenger-worker
volumes:
- ./secrets/pki/worker:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- faults:/faults
networks: [data, secrets, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
openbao: *bao-ready
otel-gate: *otel-depends
<<: *hardening
tmpfs: ["/tmp:size=256m,mode=1777"]
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 70s
deploy:
resources:
limits: {cpus: "2.00", memory: 1536M, pids: 128}
lb:
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:8443:8443"
- "127.0.0.1:8404:8404"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/pki/ingress:/run/tls-ingress:ro
- ./secrets/pki/lb:/run/tls-lb:ro
- ./secrets/pki/public:/run/tls-ca:ro
networks: [ingress, frontend]
depends_on:
api: {condition: service_healthy}
<<: *hardening
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/lb-health"]
interval: 5s
timeout: 3s
retries: 3
start_period: 10s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 128M, pids: 256}
# ---------------- наблюдаемость ----------------
# otel-gate владеет network namespace; Collector живёт в нём же и слушает только 127.0.0.1.
# Имя otel-collector в сети telemetry указывает на gate (alias): приложение ходит на https://otel-collector:4318.
otel-gate:
image: haproxy:3.2.25-alpine
volumes:
- ./infra/otel/gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/pki/otel-collector:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
networks:
telemetry:
aliases: [otel-collector]
<<: *hardening
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/gate-health"]
interval: 5s
timeout: 3s
retries: 3
start_period: 5s
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 64M, pids: 256}
otel-collector:
image: otel/opentelemetry-collector-contrib:0.164.0
command: ["--config=/etc/otelcol-contrib/config.yaml"]
network_mode: "service:otel-gate"
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
- ./secrets/pki/otel-collector:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
depends_on:
otel-gate: {condition: service_started}
<<: *hardening
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M, pids: 256}
# Одноразовый chown volumes Loki/Tempo для uid 10001 (их процессы — не root). Исключение: root + CAP_CHOWN,
# без сети, завершается сразу. Идемпотентно: при повторном up каталоги уже 10001 и внутри — 0700-каталоги
# Loki/Tempo, в которые root без CAP_DAC_READ_SEARCH не войдёт, поэтому chown только для ещё чужого volume.
telemetry-init:
image: busybox:1.37.0
user: root
command: ["sh", "-c", "for d in /loki /tempo; do [ \"$$(stat -c %u $$d)\" = 10001 ] || chown -R 10001:10001 $$d || exit 1; done"]
volumes:
- lokidata:/loki
- tempodata:/tempo
network_mode: none
cap_drop: [ALL]
cap_add: [CHOWN]
security_opt: ["no-new-privileges:true"]
read_only: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 32M, pids: 256}
prometheus:
image: prom/prometheus:v3.15.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --web.config.file=/etc/prometheus/web.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro
- ./infra/prometheus/promtool-http.yml:/etc/prometheus/promtool-http.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- ./infra/prometheus/tests:/etc/prometheus/tests:ro
- ./secrets/pki/prometheus:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- promdata:/prometheus
ports:
- "127.0.0.1:9090:9090"
networks: [telemetry, ops]
<<: *hardening
# promtool test rules (unit-тесты алертов, check_configs.py) создаёт временную TSDB в /tmp
tmpfs: ["/tmp:size=64m,mode=1777"]
healthcheck:
test: ["CMD", "promtool", "check", "ready", "--url=https://localhost:9090", "--http.config.file=/etc/prometheus/promtool-http.yml"]
interval: 5s
timeout: 3s
retries: 24
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
loki:
image: grafana/loki:3.7.9
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- ./secrets/pki/loki:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- lokidata:/loki
ports:
- "127.0.0.1:3100:3100"
networks: [telemetry, ops]
depends_on:
telemetry-init: {condition: service_completed_successfully}
<<: *hardening
tmpfs: ["/tmp:size=64m,mode=1777"]
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
tempo:
image: grafana/tempo:3.1.1
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- ./secrets/pki/tempo:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- tempodata:/var/tempo
ports:
- "127.0.0.1:3200:3200"
networks: [telemetry, ops]
depends_on:
telemetry-init: {condition: service_completed_successfully}
<<: *hardening
tmpfs: ["/tmp:size=64m,mode=1777"]
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
grafana:
image: grafana/grafana:12.4.14
environment:
GF_SERVER_PROTOCOL: https
GF_SERVER_CERT_FILE: /run/tls/tls.crt
GF_SERVER_CERT_KEY: /run/tls/tls.key
GF_SERVER_DOMAIN: localhost
GF_SERVER_ROOT_URL: https://localhost:3000/
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_SECURITY_COOKIE_SECURE: "true"
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
- ./secrets/pki/grafana:/run/tls:ro
- ./secrets/pki/public:/run/tls-ca:ro
- grafanadata:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
networks: [telemetry, ops]
depends_on:
prometheus: {condition: service_healthy}
loki: {condition: service_started}
tempo: {condition: service_started}
<<: *hardening
tmpfs: ["/tmp:size=64m,mode=1777"]
healthcheck:
test: ["CMD-SHELL", "curl -fsS -m 2 -o /dev/null --cacert /run/tls-ca/ca.crt https://localhost:3000/api/health || wget -q -T 2 -O /dev/null https://localhost:3000/api/health"]
interval: 5s
timeout: 3s
retries: 24
start_period: 20s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M, pids: 256}
# ---------------- только для проверок (профиль tools, сам не стартует) ----------------
# Контейнер-зонд во всех внутренних сетях: отрицательные/положительные проверки trust matrix изнутри
# (student-tests/tls_probe.py). Получает ТОЛЬКО тестовые сертификаты и сертификаты клиентов, которыми
# пробует войти «не туда»; ключа CA и секретов OpenBao у него нет.
probe:
build: *app-build
profiles: [tools]
command: ["python", "/probe/tls_probe.py", "matrix"]
environment:
PKI: /pki
PYTHONPATH: /app
volumes:
- ./student-tests:/probe:ro
- ./secrets/pki/public:/pki/public:ro
- ./secrets/pki/test:/pki/test:ro
- ./secrets/pki/lb:/pki/lb:ro
- ./secrets/pki/api:/pki/api:ro
- ./secrets/pki/worker:/pki/worker:ro
- ./secrets/pki/dispatcher:/pki/dispatcher:ro
- ./secrets/pki/janitor:/pki/janitor:ro
- ./secrets/pki/grafana:/pki/grafana:ro
- ./secrets/pki/otel-collector:/pki/otel-collector:ro
- ./secrets/pki/operator:/pki/operator:ro
networks: [frontend, data, secrets, telemetry]
<<: *hardening
tmpfs: ["/tmp:size=16m,mode=1777"]
restart: "no"
deploy:
resources:
limits: {cpus: "0.50", memory: 256M, pids: 256}
networks:
ingress: {}
frontend:
internal: true
data:
internal: true
secrets:
internal: true
telemetry:
internal: true
# Только для публикации админ-интерфейсов на 127.0.0.1 (internal-сеть не может публиковать порты).
ops: {}
volumes:
openbaodata: {}
pgdata: {}
redisdata: {}
s3data: {}
rabbitmqdata: {}
faults: {}
promdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
secrets:
postgres_password: {file: ./secrets/postgres_password}
s3_config: {file: ./secrets/s3_config.json}
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
grafana_admin_password: {file: ./secrets/grafana_admin_password}
+38 -20
View File
@@ -1,32 +1,43 @@
# Матрица доверия — заполнить в решении lab7
# Матрица доверия — решение lab7 (Бабиченко, БВТ2303)
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
**Общее для всех строк.** Один учебный корневой CA оператора (`scripts/pki.py`, ECDSA P-256, 365 суток; ключ CA — только `secrets/pki/ca/ca.key` у оператора, ни в один контейнер не монтируется). Leaf-сертификаты сервисов — 30 суток, identity = Subject CN (+ URI SAN `spiffe://messenger.lab/<cn>`), DNS SAN = имя сервиса в compose. Отзыв — CRL `secrets/pki/public/crl.pem` (подписан CA, nextUpdate 30 суток). Каждый контейнер монтирует только **свой** каталог `secrets/pki/<identity>` и публичный `secrets/pki/public` (read-only). Клиенты проверяют цепочку до CA и **имя сервера**; в коде нет ветки «без проверки». Отрицательные проверки каждой строки выполняет `python student-tests/test_tls_matrix.py` (контейнер `probe` во всех внутренних сетях, только тестовые и «чужие» клиентские сертификаты, без ключа CA и без секретов) — результат `evidence/tls-matrix.json`; права identity — `python student-tests/test_least_privilege.py` (`evidence/least-privilege.json`).
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
| Клиент → ingress/LB | **127.0.0.1:8443**, HTTPS (HAProxy 3.2, сертификат `ingress`: SAN localhost/127.0.0.1, TLS 1.2+, без session tickets), HSTS; бизнес-доступ — Bearer (сессия 256 бит, в Redis только SHA-256). HTTP :8080 больше не слушается | Expiry/revocation сессии: абсолютный TTL в Redis (`SESSION_TTL_SECONDS`), logout = DEL ключа, одинаково на всех репликах. Общий для реплик rate limit входа/регистрации в Redis (10 попыток/мин на username, 600/мин на IP) → 429 + Retry-After до Argon2id; бюджет хешей реплики → 503 + Retry-After | `infra/haproxy.cfg` (frontend api_in); `make test-security CA_FILE=...` (пустой trust store → ошибка проверки), `test_lab7_security.py`: чужой CA и неверное имя → `SSLCertVerificationError`, `test_login_rate_limit_is_shared`, `test_logout_revokes_token`; `test_expiry.py` (lab2) |
| LB → API (все реплики) | **api:8443**, mTLS: сервер — `api` (SAN api, HAProxy `verify required ca-file crl-file verifyhost api`), клиент — leaf `lb`. API (uvicorn, `tlsutil.py`): CERT_REQUIRED + CA (цепочка, срок), затем **allowlist CN=lb** (`API_ALLOWED_CLIENTS`) и CRL до чтения запроса, иначе abort. Порт API не опубликован, сеть `frontend` internal; health-listener 127.0.0.1:8081 только внутри контейнера, только /health/* | Ротация leaf `lb`: `pki.py rotate lb` → `docker compose kill -s USR2 lb` (reload без обрыва) → `pki.py revoke lb --previous`: API перечитывает CRL каждые 5 с (подпись CRL проверяется ключом CA) и **закрывает уже открытые соединения** с отозванным сертификатом; session tickets выключены | `tls_probe.py lb-api`: нет сертификата / чужой CA с CN=lb / истёкший lb / intruder (наш CA) / worker / api → отказ; lb → 200; клиент, доверяющий только чужому CA → не принимает сервер; plaintext → нет ответа. `lifecycle.py rotate-leaf` (старый leaf → отказ, новый → 200, нагрузка без ошибок) |
| API/worker → PostgreSQL | **postgres:5432**, TLS (`ssl=on`, TLS 1.2+), клиенты `sslmode=verify-full` + CA (имя postgres); `pg_hba`: только `hostssl … scram-sha-256`, `hostnossl … reject`. Роли: `messenger_api`, `messenger_worker`, `messenger_dispatcher`, `messenger_janitor` (GRANT по запросам, `migrations/006_roles.sql`), владелец `messenger` — только migrate | Пароли ролей — из OpenBao (`messenger/<сервис>.db_password`), берутся на каждое новое соединение пула; migrate выставляет их как SCRAM-verifier (открытый пароль не попадает ни в SQL, ни в журнал сервера). Ротация пароля роли (процедура, не демонстрировалась — на 4 показана ротация пароля Redis): новая версия в KV → `docker compose run --rm migrate` выставляет её роли → сервисы перечитывают KV, новые соединения — с новым паролем; окно до 30 с, когда новые соединения со старым паролем получают отказ (пул держит открытые). mTLS для PostgreSQL не включён (допустимое усиление, не требование) | `infra/postgres/pg_hba.conf`, `dbconn.py`; `tls_probe.py postgres`: verify-full → дошли до SCRAM (TLS ок), `sslmode=disable` → `no pg_hba.conf entry … no encryption`, доверие чужому CA / подключение по IP → отказ проверки сервера. `test_least_privilege.py`: worker не читает users/messages, api не DELETE/DROP/CREATE/ALTER ROLE и т.д. |
| API/worker → S3 | **s3:8333** HTTPS (SeaweedFS 4.52 от uid 1000, сертификат s3; одновременных тел загрузок ≤ 128 MiB, остальные в очереди); master/volume/filer слушают только 127.0.0.1 контейнера; Iceberg/Lance/IAM API выключены; gRPC шлюза :18333 — mTLS, только CN=s3 (`infra/seaweedfs/security.toml`). Ключи: s3-init (admin), api, worker, janitor — разные, из OpenBao | Права по prefix (`s3_config`): api — PUT/DELETE `staging/`, GET `attachments/`,`originals/`; worker — GET `staging/`,`originals/`, PUT/GET/LIST `attachments/`, DELETE исходника; janitor — LIST и DELETE, без GET; никто кроме s3-init не создаёт bucket | `tls_probe.py s3`: аноним по HTTPS → 403, чужой CA → отказ проверки, plaintext → 400 «HTTP to HTTPS», gRPC :18333 с CN=intruder/api/без сертификата → bad certificate / certificate required. `test_least_privilege.py`: api → PUT attachments/, GET staging/, LIST → AccessDenied; worker → LIST корня, PUT вне prefix → AccessDenied; janitor → GET → AccessDenied |
| API/worker → broker | **rabbitmq:5671** AMQPS (порт 5672 выключен), `verify_peer` + `fail_if_no_peer_cert`; вход механизмом **EXTERNAL**: пользователь = CN сертификата (`worker`, `dispatcher`), паролей нет; права: worker — read основной очереди, dispatcher — write в exchange + read DLQ, configure ни у кого. У API брокера нет (outbox) | mTLS с ограничением identity: сертификат нашего CA с CN без пользователя брокера (api, lb, intruder) → ACCESS_REFUSED при входе; worker не публикует, dispatcher не читает задания, никто не объявляет очереди. Erlang distribution и epmd — только 127.0.0.1 | `infra/rabbitmq.conf`, `rabbitmq-enabled-plugins`; `tls_probe.py broker`: worker/dispatcher → вход, api/intruder/lb → login refused, чужой CA / без сертификата → отказ TLS, worker publish / dispatcher basic_get / queue_declare → 403 |
| API/worker → Collector | **otel-collector:4318** (alias otel-gate) OTLP/HTTP по mTLS: шлюз HAProxy проверяет CA, срок, CRL и **CN ∈ {api, worker, dispatcher, janitor}**, иначе 403; Collector слушает только 127.0.0.1:4319 в общем network namespace со шлюзом (локальный hop не виден из сети) | Раздельные identity: каждый сервис шлёт со своим сертификатом (`OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`); сертификат того же CA, но не приложения (lb, grafana, operator, intruder) → 403. Метрики самого Collector — :8889 через шлюз только для CN prometheus/operator | `infra/otel/gate.cfg`, `infra/otel/collector.yaml`; `tls_probe.py collector`: 4 app-identity → 200, lb/grafana/operator/intruder → 403, чужой CA / без сертификата → отказ TLS, :8889 с CN=api → 403 |
| Collector → telemetry backends | Collector → Tempo :4317 (OTLP gRPC, TLS, `client_ca_file`), → Loki :3100 и Prometheus :9090 (HTTPS); везде клиентский сертификат CN=otel-collector и проверка сервера по CA и имени. Prometheus scrape — тоже HTTPS + сертификат prometheus | Prometheus: `client_allowed_sans` — пишут/читают только otel-collector, grafana, prometheus, operator (api, intruder того же CA → отказ на рукопожатии). Loki/Tempo различают только «наш CA / не наш» — разделить чтение и запись по identity они не умеют (остаточный риск, сеть telemetry internal). Ротация leaf otel-collector: Prometheus перечитывает web-config на каждый запрос; Collector/Loki/Tempo — рестарт сервиса | `infra/prometheus/web.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` (в т.ч. внутренний gRPC Tempo по TLS), `tls_probe.py telemetry` |
| Grafana → data sources | HTTPS к Prometheus/Loki/Tempo с проверкой CA (`tlsAuthWithCACert`) и клиентским сертификатом CN=grafana (`$__file{}` в provisioning, в Git сертификатов нет) | Отрицательные проверки: без сертификата / чужой CA к Loki/Tempo/Prometheus → отказ; к Prometheus — только разрешённые SAN | `infra/grafana/provisioning/datasources/datasources.yaml`; `check_configs.py` (health источников = OK), `tls_probe.py telemetry` |
| API/worker/agent → OpenBao | **openbao:8200** TLS (сертификат openbao), вход **cert auth**: роль = CN, `allowed_common_names=[роль]`, доверие только нашему CA, CRL загружен в cert auth. Токен: service, TTL 15 мин, renewable, max 1 ч; policy — read только `messenger/data/<роль>` | Renewal: `secretstore.py` продлевает токен (renew-self с тем же сертификатом), при отказе/исчерпании max TTL — новый вход сертификатом; KV перечитывается раз в 30 с (подхват ротации). Отзыв: `bao.py revoke-service-tokens` + CRL в cert auth (отозванный leaf не войдёт). Sealed/недоступен: работающий процесс живёт на секретах в памяти, новый — не стартует (fail closed) | `infra/openbao/config.hcl`, `infra/openbao/policies/*.hcl`, `scripts/bao.py`; `tls_probe.py openbao`: worker как api / intruder / чужой CA / без сертификата → 400; `test_least_privilege.py`: чужие пути, запись KV/политики, список auth, создание токена → 403; `lifecycle.py token-expiry`, `lifecycle.py sealed` |
| Оператор → OpenBao/Grafana/S3 admin | OpenBao: 127.0.0.1:8200 HTTPS + root token оператора (файл `secrets/operator/openbao-init.json`, 0600, вне Git и контейнеров). Grafana: https://localhost:3000, admin + пароль. Prometheus/Loki/Tempo: 127.0.0.1, mTLS сертификатом `operator`. HAProxy stats: 127.0.0.1:8404, mTLS, только CN=operator. Admin-ключ S3 — только у s3-init через OpenBao, консоли S3 нет | Роли разделены: приложение не имеет operator-прав нигде (нет root token, admin S3, владельца БД, admin Redis); оператор не использует ключи сервисов. Redis admin — только у оператора для ротации (`lifecycle.py`) | `check_stack.py`/`check_hardening.py`: опубликованы только 8443, 8404, 8200, 9090, 3100, 3200, 3000 и только на 127.0.0.1; `tls_probe.py telemetry` (без сертификата оператора → отказ) |
| API → Redis/Valkey, если используется | **redis:6379** только TLS (`port 0`), **tls-auth-clients yes** (клиентский сертификат нашего CA обязателен), затем ACL: `default off`, `api` — только ключи `session:*`, `ratelimit:*` и команды get/set/del/incr/incrby/pexpire/pttl/ping/multi/exec, без pub/sub; `admin` — оператор | Ротация пароля `api` без рестартов: временно два пароля (`ACL SETUSER api >new`) → новая версия в OpenBao → все реплики прочитали → `ACL SETUSER api resetpass >new` + `CLIENT KILL USER api` + `ACL SAVE`. mTLS включён (identity на уровне TLS — любой сертификат нашего CA, права — ACL) | `infra/redis.conf`, `scripts/init_secrets.py` (ACL), `auth.py`; `tls_probe.py redis`: наш CA без AUTH → NOAUTH, неверный пароль → WRONGPASS, без сертификата / чужой CA → отказ TLS, plaintext → сброс; `test_least_privilege.py`: KEYS, FLUSHALL, CONFIG, ACL, PUBLISH, чужие ключи → NOPERM; `lifecycle.py rotate-secret` |
| Replication/кластерные каналы, если добавлены | N/A: репликации и кластеров нет (по одному PostgreSQL, Redis, RabbitMQ, SeaweedFS). Внутрипроцессные каналы закрыты: SeaweedFS master/volume/filer — 127.0.0.1 + gRPC mTLS (CN=s3); RabbitMQ distribution/epmd — 127.0.0.1; Loki gRPC — 127.0.0.1; Tempo внутренний gRPC — TLS с клиентским сертификатом | Модель отказа: один Docker-host, отказ узла не выдерживается (см. REPORT.md, ограничения) | `tls_probe.py s3` (gRPC 18333), `infra/seaweedfs/security.toml`, `infra/rabbitmq.conf` |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
## Служебные каналы (scrape / health / bootstrap)
| Канал | Как защищён | Почему не обход |
| --- | --- | --- |
| Healthcheck API | `http://127.0.0.1:8081/health/{live,ready}` — отдельный listener **внутри контейнера** | loopback контейнера недоступен из сетей; бизнес-путей нет (404) |
| Healthcheck LB / otel-gate | `127.0.0.1:8405` внутри контейнера (monitor-uri) | то же |
| Healthcheck PostgreSQL / Redis / RabbitMQ / SeaweedFS | unix-socket `pg_isready`; `redis-cli --tls` без AUTH (ответ NOAUTH); `rabbitmq-diagnostics` по loopback-distribution; `wget http://127.0.0.1:9333/dir/assign` (master на loopback; готовность к записи, а не только чтению — после рестарта мастер ~15 с не выдаёт fid) | всё внутри контейнера |
| Healthcheck Prometheus / Grafana / OpenBao | `promtool check ready --http.config.file` (mTLS), `curl --cacert https://localhost:3000`, `bao status -ca-cert` | TLS и внутри контейнера |
| Prometheus scrape | HTTPS + сертификат prometheus к Collector (через gate, CN-allowlist), Loki, Tempo, себе | см. строку Collector → backends |
| Bootstrap | migrate (CN=migrate: пароль владельца БД + пароли ролей), s3-init (CN=s3-init: admin S3) — одноразовые, свои политики OpenBao; оператор — `pki.py`, `init_secrets.py`, `bao.py` на хосте | значения не печатаются; файл с секретами приложений удаляется после загрузки в OpenBao |
## Обязательные отрицательные проверки
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». → `make test-security`, `test_lab7_security.py test_https_only_and_certificate_checks`.
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. → `test_tls_matrix.py lb-api` (+ истёкший lb, сертификат самого api, отозванный предыдущий leaf).
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. → `test_tls_matrix.py broker collector` (identity проверяется самим брокером — EXTERNAL/permissions — и шлюзом Collector по CN).
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. → `test_lab7_security.py test_outsider_cannot_reach_foreign_chat`, `test_api.py` (lab2).
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. → `test_least_privilege.py` (openbao: 5 чужих путей, запись KV, политика, список auth, токен; «own secret keys (no admin credentials)»).
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. → `lifecycle.py rotate-leaf` (новое подключение старым leaf — отказ; API закрывает открытые соединения со старым leaf; tickets выключены), `lifecycle.py rotate-secret` (`CLIENT KILL USER api` + проверка, что активен только новый пароль), `lifecycle.py token-expiry`.
## Bootstrap и эксплуатация
@@ -35,3 +46,10 @@
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
**Решение.**
- *Кто владеет чем.* Оператор (студент на своём хосте) — ключ CA (`secrets/pki/ca/`), доли unseal и root token (`secrets/operator/openbao-init.json`), пароль admin Redis и владельца БД (`secrets/operator/`). Каталог `secrets/` 0700, файлы оператора 0600, ни один не смонтирован в контейнер. В реальной эксплуатации: `bao.py init --print-shares-once` раздаёт 3 доли разным людям (порог 2) и ничего не сохраняет, `bao.py unseal --prompt`; root token отзывается после настройки; CA — офлайн.
- *Первый секрет сервиса* — его leaf-сертификат (30 суток, отзывается CRL, меняется без пересборки образа). Постоянного «секрета для получения секретов» (SecretID/BAO_TOKEN в env) нет: процесс входит в OpenBao методом cert и получает токен на 15 мин.
- *Рестарт API* — без участия оператора (сертификат на месте, OpenBao распечатан). *Рестарт OpenBao* — он sealed: `python scripts/bao.py unseal` (restart-hook делает это сам, читая доли из файла оператора; ничего не печатается). Пока sealed: работающие сервисы обслуживают запросы с секретами из памяти, новые/перезапущенные не становятся ready.
- *Renewal.* Leaf — `pki.py rotate <identity>` + reload/restart сервиса (каталоги смонтированы, образы не меняются), старый — `pki.py revoke <identity> --previous` + `bao.py sync-crl` + reload lb/otel-gate; CRL переподписывается `pki.py crl` раньше nextUpdate (30 суток). Токены — автоматически (renew/relogin). Рабочие секреты — новая версия KV, сервисы перечитывают её сами.
- *Восстановление.* Потеря OpenBao volume = потеря секретов приложений → `init_secrets.py` после удаления `secrets/s3_config.json`/`redis/users.acl` и пересоздание volumes (учебные данные). Потеря CA → новый CA и все leaf (`pki.py init` в чистом `secrets/pki`), `bao.py configure` (роли cert auth получают новый CA).
+464
View File
@@ -0,0 +1,464 @@
import logging
import time
from contextlib import contextmanager
from datetime import datetime
from uuid import UUID, uuid4
import psycopg
from psycopg import errors as pg_errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
import dbconn
import pipeline
import settings
import telemetry
from errors import bad_request, conflict, not_found, unavailable
log = logging.getLogger("messenger.db")
def check_connection(conn) -> None:
"""Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса."""
from opentelemetry.instrumentation.utils import suppress_instrumentation
with suppress_instrumentation():
ConnectionPool.check_connection(conn)
def pool_kwargs() -> dict:
"""lab7: вызывается на КАЖДОЕ новое соединение пула — пароль роли берётся из OpenBao (текущая версия)."""
return dbconn.connect_kwargs(
row_factory=dict_row,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
)
pool = ConnectionPool(
conninfo="",
min_size=settings.DB_POOL_MIN,
max_size=settings.DB_POOL_MAX,
timeout=settings.DB_POOL_WAIT_SECONDS,
kwargs=pool_kwargs,
check=check_connection,
open=False,
)
# lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»).
POOL_NAME = {"db.client.connection.pool.name": "messenger-api"}
POOL_WAIT = telemetry.meter.create_histogram(
"db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool")
def _pool_stats():
try:
return pool.get_stats()
except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет
return None
def _observe_connections(_options):
from opentelemetry.metrics import Observation
stats = _pool_stats()
if not stats:
return []
size, available = stats.get("pool_size", 0), stats.get("pool_available", 0)
return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}),
Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})]
def _observe_stat(key):
def callback(_options):
from opentelemetry.metrics import Observation
stats = _pool_stats()
return [Observation(stats.get(key, 0), POOL_NAME)] if stats else []
return callback
telemetry.meter.create_observable_up_down_counter(
"db.client.connection.count", [_observe_connections], unit="{connection}",
description="Connections in the pool by state (used/idle)")
telemetry.meter.create_observable_up_down_counter(
"db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}",
description="Maximum pool size (DB_POOL_MAX)")
telemetry.meter.create_observable_up_down_counter(
"db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}",
description="Requests waiting for a free connection now")
telemetry.meter.create_observable_counter(
"db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}",
description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS")
@contextmanager
def connection(wait_seconds: float | None = None):
try:
started = time.perf_counter()
try:
with pool.connection(timeout=wait_seconds) as conn:
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
yield conn
except PoolTimeout:
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
raise
except (psycopg.OperationalError, PoolTimeout) as exc:
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__,
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
raise unavailable() from None
except psycopg.DataError as exc:
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__,
extra={"event": "db.rejected_value", "error.type": type(exc).__name__})
raise bad_request("Invalid request data") from None
TEXT_FIELDS = ("display_name", "title", "text", "filename")
def to_db(value: str) -> bytes:
return value.encode("utf-8")
def from_db(row: dict | None) -> dict | None:
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
if row is None:
return None
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
def ping() -> None:
with connection(wait_seconds=1) as conn:
conn.execute("SELECT 1")
MIGRATIONS_LOCK = 20260202
def migration_files() -> list:
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
def run_migrations() -> list[str]:
files = migration_files()
applied = []
with dbconn.connect(autocommit=True) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
try:
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now())""")
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
for path in files:
if path.name in done:
continue
with conn.transaction():
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
applied.append(path.name)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
return applied
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
deadline = time.monotonic() + max_wait_seconds
while True:
try:
applied = run_migrations()
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
return
except psycopg.OperationalError:
if time.monotonic() > deadline:
raise
log.warning("PostgreSQL is not ready yet, retry in 1 s")
time.sleep(1)
def missing_migrations() -> list[str]:
expected = [path.name for path in migration_files()]
with dbconn.connect(autocommit=True) as conn:
try:
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
except (pg_errors.UndefinedTable, pg_errors.InsufficientPrivilege):
# lab7: права ролей выдаёт миграция 006 — до неё схема для приложения «не готова».
return expected
return [name for name in expected if name not in done]
def wait_for_schema(max_wait_seconds: int) -> None:
deadline = time.monotonic() + max_wait_seconds
while True:
try:
missing = missing_migrations()
except psycopg.OperationalError:
missing = None
if missing == []:
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
return
if time.monotonic() > deadline:
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
else "missing " + ", ".join(missing))
time.sleep(1)
USER_COLUMNS = "id, username, display_name, created_at"
def create_user(username: str, display_name: str, password_hash: str) -> dict:
try:
with connection() as conn:
return from_db(conn.execute(
f"""INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
(uuid4(), username, to_db(display_name), password_hash),
).fetchone())
except pg_errors.UniqueViolation:
raise conflict("Username already exists") from None
def get_user(user_id: UUID) -> dict | None:
with connection() as conn:
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
def get_user_with_password(username: str) -> dict | None:
with connection() as conn:
return from_db(conn.execute(
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
).fetchone())
def update_password_hash(user_id: UUID, old_hash: str, new_hash: str) -> bool:
"""lab7: rehash при входе (параметры Argon2id выросли). Условие на старый hash — без гонки с другим входом."""
with connection() as conn:
cursor = conn.execute("UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s",
(new_hash, user_id, old_hash))
return cursor.rowcount == 1
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
def create_chat(title: str, member_ids: list[UUID]) -> dict:
chat_id = uuid4()
members = sorted(member_ids)
try:
with connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
(chat_id, to_db(title)),
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat_id, member_id) for member_id in members],
)
except pg_errors.ForeignKeyViolation:
raise bad_request("Unknown member") from None
return {**from_db(row), "member_ids": members}
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
with connection() as conn:
chat = conn.execute(
f"""SELECT {CHAT_COLUMNS} FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
WHERE c.id = %s""",
(user_id, chat_id),
).fetchone()
if chat is None:
raise not_found("Chat not found")
return from_db(chat)
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
where, params = "", [user_id]
if after is not None:
where = "WHERE (c.created_at, c.id) > (%s, %s)"
params += list(after)
with connection() as conn:
rows = conn.execute(
f"""SELECT {CHAT_COLUMNS} FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
{where}
ORDER BY c.created_at, c.id
LIMIT %s""",
(*params, limit + 1),
).fetchall()
return [from_db(row) for row in rows]
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
).fetchone() is not None
def check_member(chat_id: UUID, user_id: UUID) -> None:
with connection() as conn:
if not is_member(conn, chat_id, user_id):
raise not_found("Chat not found")
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
idempotency_key: str, request_hash: bytes) -> dict:
try:
with connection() as conn:
with conn.transaction():
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
raise not_found("Chat not found")
if attachment_ids:
# Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409).
# FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние.
found = conn.execute(
"""SELECT id, state FROM attachments
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""",
(attachment_ids, chat_id),
).fetchall()
if len(found) != len(attachment_ids):
raise not_found("Attachment not found")
failed = any(row["state"] == "failed" for row in found)
else:
failed = False
saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s"""
if failed:
# Повтор уже созданного сообщения возвращает прежний 201, даже если вложение
# с тех пор стало failed. Новое сообщение с failed-вложением — 409.
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
if saved is None:
raise conflict("Failed attachment cannot be attached")
created = None
else:
created = conn.execute(
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
RETURNING id, chat_id, sender_id, text, created_at""",
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
).fetchone()
if created is None:
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
if created is not None:
if attachment_ids:
with conn.cursor() as cur:
cur.executemany(
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES (%s, %s, %s, %s)""",
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
)
return {**from_db(created), "attachment_ids": list(attachment_ids)}
except pg_errors.ForeignKeyViolation:
raise not_found("Chat not found") from None
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
raise unavailable()
if bytes(saved.pop("request_hash")) != request_hash:
raise conflict("Idempotency-Key was already used with a different request")
return from_db(saved)
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
where, params = "m.chat_id = %s", [chat_id]
if after is not None:
where += " AND (m.created_at, m.id) > (%s, %s)"
params += list(after)
with connection() as conn:
if not is_member(conn, chat_id, user_id):
raise not_found("Chat not found")
rows = conn.execute(
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
WHERE {where}
ORDER BY m.created_at, m.id
LIMIT %s""",
(*params, limit + 1),
).fetchall()
return [from_db(row) for row in rows]
# ---------- вложения ----------
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, "
"pipeline_version, created_at, updated_at, object_key")
VISIBLE_STATES = ("queued", "processing", "ready", "failed")
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None:
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging.
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД).
"""
try:
with connection() as conn:
conn.execute(
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, source_key, state)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
width, height, source_key),
)
except pg_errors.ForeignKeyViolation:
raise not_found("Chat not found") from None
def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None:
"""Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией.
После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL,
сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace).
"""
with connection() as conn:
with conn.transaction():
row = conn.execute(
f"""UPDATE attachments SET state = 'queued', updated_at = now()
WHERE id = %s AND state = 'uploading'
RETURNING {ATTACHMENT_COLUMNS}""",
(attachment_id,),
).fetchone()
if row is None:
return None
pipeline.enqueue_new(conn, attachment_id, actor)
return from_db(row)
def delete_upload_intent(attachment_id: UUID) -> None:
with connection(wait_seconds=1) as conn:
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
"""Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404.
Возвращает строку и варианты текущей версии pipeline (они есть только у ready).
"""
with connection() as conn:
row = conn.execute(
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
FROM attachments a
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
WHERE a.id = %s AND a.state = ANY(%s)""",
(user_id, attachment_id, list(VISIBLE_STATES)),
).fetchone()
if row is None:
raise not_found("Attachment not found")
variants = []
if row["state"] == "ready":
variants = conn.execute(
"""SELECT name, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s
ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""",
(attachment_id, row["pipeline_version"]),
).fetchall()
return {**from_db(row), "variants": variants}
+33
View File
@@ -0,0 +1,33 @@
"""Параметры подключения к PostgreSQL (lab7) — общие для api, worker, dispatcher, janitor, ops, migrate.
TLS verify-full: цепочка до учебного CA и имя сервера = DB_HOST (SAN сертификата postgres). На сервере
pg_hba разрешает только hostssl + scram-sha-256, незашифрованное подключение отвергается.
Роль у каждого сервиса своя (DB_USER: messenger_api, messenger_worker, ...), права — migrations/006_roles.sql.
Пароль берётся из OpenBao при КАЖДОМ новом подключении: после ротации новые соединения пула используют
новую версию, уже открытые продолжают работать до закрытия.
"""
import psycopg
import settings
from secretstore import store
def connect_kwargs(**extra) -> dict:
params = {
"host": settings.DB_HOST,
"port": settings.DB_PORT,
"dbname": settings.DB_NAME,
"user": settings.DB_USER,
"password": store.get(settings.DB_PASSWORD_KEY),
"sslmode": settings.DB_SSLMODE,
"sslrootcert": settings.TLS_CA_FILE,
"sslcertmode": "disable", # клиентский сертификат PostgreSQL не нужен (аутентификация — SCRAM)
"application_name": f"{settings.BAO_ROLE}@{settings.INSTANCE_ID}",
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
}
params.update(extra)
return params
def connect(**kwargs) -> psycopg.Connection:
return psycopg.connect(**connect_kwargs(), **kwargs)
+265
View File
@@ -0,0 +1,265 @@
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
Цикл:
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
"""
import logging
import signal
import threading
import time
from pathlib import Path
import psycopg
from opentelemetry.trace import SpanKind
from psycopg.rows import dict_row
import dbconn
import secretstore
import settings
import telemetry
telemetry.setup("messenger-dispatcher")
import broker # noqa: E402
import faults # noqa: E402
import pipeline # noqa: E402
log = logging.getLogger("messenger.dispatcher")
# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs,
# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому
# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик
# только читает последний снимок (не ходит в БД из потока экспорта).
STATS_SECONDS = 5
STATS = {"at": 0.0, "db": None, "broker": None}
PUBLISHED = telemetry.meter.create_counter(
"messenger.messages.published", unit="{message}",
description="Outbox rows published to the broker (outcome: ok/error)")
QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE}
def _fresh(kind):
return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None
def _observe(kind, build):
def callback(_options):
from opentelemetry.metrics import Observation
data = _fresh(kind)
# lab7: ресурс, невидимый identity (RabbitMQ 4.2+), даёт None — такую точку не пишем
return [Observation(value, attrs) for value, attrs in build(data) if value is not None] if data else []
return callback
telemetry.meter.create_observable_gauge(
"messenger.queue.depth", [_observe("broker", lambda d: [
(d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])],
unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)")
telemetry.meter.create_observable_gauge(
"messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])],
unit="{consumer}", description="Consumers subscribed to the queue")
telemetry.meter.create_observable_gauge(
"messenger.jobs.backlog", [_observe("db", lambda d: [
(d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}),
(d["processing"], {"status": "processing"})])],
unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing")
telemetry.meter.create_observable_gauge(
"messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])],
unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits")
telemetry.meter.create_observable_gauge(
"messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])],
unit="{message}", description="Outbox rows due but not yet published")
telemetry.meter.create_observable_gauge(
"messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])],
unit="s", description="Age of the oldest due unpublished outbox row (publish lag)")
def collect_stats(conn, publisher) -> None:
"""Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот."""
if time.monotonic() - STATS["at"] < STATS_SECONDS:
return
db_stats = conn.execute(
"""SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due,
count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed,
count(*) FILTER (WHERE status = 'processing') AS processing,
coalesce(extract(epoch FROM now() - min(available_at)
FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age
FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone()
outbox = conn.execute(
"""SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due,
coalesce(extract(epoch FROM now() - min(available_at)
FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age
FROM outbox WHERE published_at IS NULL""").fetchone()
STATS["db"] = {**db_stats, **outbox}
try:
STATS["broker"] = broker.check_topology(publisher.open())
except broker.BrokerError:
STATS["broker"] = None
publisher.close()
STATS["at"] = time.monotonic()
STOP = threading.Event()
ACTOR = "dispatcher@" + settings.INSTANCE_ID
def connect_db():
# lab7: TLS verify-full, своя роль PostgreSQL, пароль из OpenBao (dbconn)
return dbconn.connect(autocommit=True, row_factory=dict_row,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def publish_due(conn, publisher: broker.Publisher) -> int:
published = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, job_id, attempt, payload FROM outbox
WHERE published_at IS NULL AND available_at <= now()
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
(settings.DISPATCHER_BATCH,),
).fetchall()
if not rows:
return published
for row in rows:
publish_one(conn, publisher, row)
published += 1
log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
if len(rows) < settings.DISPATCHER_BATCH:
return published
return published
def publish_one(conn, publisher: broker.Publisher, row: dict) -> None:
"""Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст
уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу."""
message = dict(row["payload"])
job_id, attempt = message.get("job_id"), message.get("attempt")
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send",
"messaging.destination.name": pipeline.EXCHANGE,
"messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY,
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
"messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]}
parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate"))
with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent,
kind=SpanKind.PRODUCER, attributes=attributes,
record_exception=False, set_status_on_exception=False) as span:
carrier = telemetry.inject()
if carrier.get("traceparent"):
message["traceparent"] = carrier["traceparent"]
message.pop("tracestate", None)
if carrier.get("tracestate"):
message["tracestate"] = carrier["tracestate"]
faults.crash("dispatcher_crash_before_publish")
try:
publisher.publish(message)
except broker.BrokerError as exc:
telemetry.mark_error(span, type(exc).__name__)
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"})
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__,
extra={"event": "outbox.publish_failed", "job.id": str(job_id),
"error.type": type(exc).__name__})
publisher.close()
raise
faults.crash("dispatcher_crash_after_confirm")
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"})
def note_failure(conn) -> None:
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
try:
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
WHERE published_at IS NULL AND available_at <= now()""")
except psycopg.Error:
pass
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def run() -> None:
publisher = broker.Publisher()
conn = None
last_leases = last_reconcile = 0.0
pause = 1.0
while not STOP.is_set():
try:
if conn is None or conn.closed:
conn = connect_db()
conn.execute("LISTEN outbox")
log.info("connected to PostgreSQL, listening for outbox notifications",
extra={"event": "db.connected"})
touch_alive()
publish_due(conn, publisher)
collect_stats(conn, publisher)
now = time.monotonic()
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
last_leases = now
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
log.warning("recovered %s job(s) with expired lease", recovered,
extra={"event": "lease.recovered", "count": recovered})
if now - last_reconcile >= 30:
last_reconcile = now
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
log.warning("re-published %s queued job(s) without a message in flight", lost,
extra={"event": "outbox.republished", "count": lost})
pause = 1.0
publisher.pump()
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
pass
except broker.BrokerError:
if conn is not None and not conn.closed:
note_failure(conn)
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause,
extra={"event": "broker.unavailable"})
STOP.wait(pause)
pause = min(pause * 2, 15)
except psycopg.Error as exc:
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause,
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
try:
if conn is not None:
conn.close()
except psycopg.Error:
pass
conn = None
STOP.wait(pause)
pause = min(pause * 2, 15)
publisher.close()
if conn is not None:
conn.close()
log.info("stopped", extra={"event": "service.stopped"})
telemetry.shutdown()
def main() -> None:
if not settings.BROKER_URL:
raise SystemExit("BROKER_URL is required")
secretstore.install_metrics(telemetry.meter)
secretstore.store.start() # lab7: пароль БД dispatcher — из OpenBao (CN=dispatcher)
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
run()
if __name__ == "__main__":
main()
+123
View File
@@ -0,0 +1,123 @@
# Runbooks алертов (lab6)
Правила — `infra/prometheus/rules/alerts.yml`, unit-тесты правил — `infra/prometheus/tests/alerts_test.yml`.
Состояние алертов: http://127.0.0.1:9090/alerts или Grafana → Alerting → Alert rules (data source rules).
Dashboard: Grafana http://127.0.0.1:3000 → Messenger → «Messenger — lab6 observability».
Общий порядок: **метрика говорит «что и когда», трасса — «где именно», лог — «почему»**.
Ссылки на трассу из лога: Grafana → Explore → Loki → запись → поле `trace_id` → «Открыть трассу».
Из спана в логи: Tempo → спан → «Logs for this span» (запрос по `trace_id`).
---
## MessengerHighErrorRate
**Что значит.** Больше 5% бизнес-запросов API (без `/health/*`) за минуту закончились 5xx, дольше минуты,
при трафике > 0,2 запроса/с. 4xx (неверный пароль, чужой чат) сюда не входят.
**Влияние.** Пользователи получают 503/500, SLO-1 (99% без 5xx за час) расходует бюджет в 5+ раз быстрее нормы.
**Найти причину.**
1. Какие маршруты и коды: dashboard «RPS по классу статуса» или
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))`.
503 — недоступна зависимость, 500 — ошибка кода.
2. Медленная БД или медленный обработчик:
- `histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))` и
`db_client_connection_pending_requests` — запросы ждут соединение из пула → соединения заняты долгими запросами;
- Tempo, TraceQL `{ resource.service.name = "messenger-api" && status = error }` → открыть трассу:
если почти всё время server-спана занимает дочерний спан `INSERT`/`SELECT` (PostgreSQL) с ошибкой —
проблема в БД; если время в `argon2id hash`/`inspect image` или «пустое» время самого server-спана — в обработчике;
`redis GET` с ошибкой — Redis; `S3 PutObject` — SeaweedFS.
3. Логи API: `{service_name="messenger-api"} | event="db.unavailable"` (поле `error_type`: `QueryCanceled` —
statement_timeout 5 с, `PoolTimeout` — нет свободного соединения 3 с, `OperationalError` — БД недоступна);
`| event="redis.unavailable"`, `| event="s3.unavailable"`.
4. В PostgreSQL (если трассы указывают на БД): `docker compose exec postgres psql -U messenger -d messenger -c
"SELECT pid, state, wait_event_type, wait_event, now()-query_start AS age, left(query,60) FROM pg_stat_activity
WHERE datname='messenger' ORDER BY age DESC"` — блокировки (`wait_event_type = Lock`) и долгие запросы.
**Исправить.** Снять блокировку/долгий запрос (`pg_terminate_backend(pid)`), восстановить зависимость
(`docker compose start postgres|redis|s3`). Не перезапускать API «на всякий случай»: причина не в нём.
**Проверить восстановление.** Доля 5xx на dashboard → 0, алерт становится inactive через ~1–2 минуты
(окно rate 1m). Пример разбора: REPORT.md, инцидент 2.
---
## MessengerQueueLagGrowing
**Что значит.** Старейшее задание обработки вложения, которое уже можно брать (статус `queued`,
`available_at` прошёл), ждёт больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно.
**Влияние.** Вложения остаются `queued`, клиенты не получают превью; SLO-3 (95% ready ≤ 30 с) нарушается.
**Найти причину.**
1. `messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}` = 0 → worker не подключён
(см. MessengerWorkerNoConsumers). > 0 → worker есть, но не успевает или падает на заданиях.
2. Насыщение worker: `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` ≈ 1 и растёт
`messenger_queue_depth` → нужно больше потоков/реплик worker (`WORKER_CONCURRENCY`, `--scale worker=2`).
3. Ошибки обработки: `sum by (outcome) (rate(messenger_job_duration_seconds_count[5m]))` — много `retry`/`failed`;
логи `{service_name="messenger-worker"} | event=~"job.retry|job.failed"` (поле `error_type`).
4. Отличить от проблемы публикации: `messenger_outbox_oldest_age_seconds` большой → сообщения не доходят до
брокера (dispatcher/RabbitMQ), а не worker; лог `{service_name="messenger-dispatcher"} | event="broker.unavailable"`.
5. Трасса любой зависшей загрузки (по `trace_id` из лога API `event="http.request"` для POST attachments):
есть `publish attachments`, но нет `process attachments.process.v1` → сообщение в брокере, его никто не взял.
**Исправить.** `docker compose start worker` (или `scripts/worker.sh start`); при насыщении — увеличить
параллелизм; при массовых ошибках — устранить их причину, затем `ops.py replay` для failed.
**Проверить восстановление.** `messenger_jobs_backlog{status="queued"}` → 0, возраст → 0, вложения `ready`,
алерт inactive через ~15–30 с. Пример: REPORT.md, инцидент 1.
---
## MessengerWorkerNoConsumers
**Что значит.** RabbitMQ дольше минуты не видит ни одного подписчика очереди `attachments.process.v1`.
**Найти причину.** `docker compose ps worker`; последние записи `{service_name="messenger-worker"}` в Loki
(`event="service.stopped"` — остановлен штатно; `event="broker.unavailable"` — не может подключиться к брокеру;
отсутствие записей и метрик `messenger_worker_capacity` — процесс не запущен).
**Исправить.** Запустить worker / восстановить RabbitMQ. Принятые задания не теряются (outbox + durable очередь).
---
## MessengerDbPoolSaturated
**Что значит.** На реплике API запросы дольше минуты ждут свободное соединение из пула (`DB_POOL_MAX`=10).
**Найти причину.** Почти всегда это следствие медленных запросов или блокировок в PostgreSQL, а не нехватки
соединений: те же шаги, что в MessengerHighErrorRate (трассы с долгими DB-спанами, `pg_stat_activity`).
Увеличивать пул имеет смысл, только если DB-спаны быстрые, а ожидание пула долгое при высоком RPS.
---
## TelemetryExportFailing
**Что значит.** Collector не может отправить данные в Tempo / Loki / Prometheus (метка `exporter`): очередь
exporter дольше минуты заполнена больше чем на 5% (хранилище не принимает), либо пакеты уже теряются —
у пакета истекло окно повторов 120 с (`send_failed`) или очередь переполнена (`enqueue_failed`). Окно повторов
считается для пакета с первой попытки, время ожидания в очереди в него не входит: очередь (200 пакетов) —
это и есть буфер на время недоступности.
**Влияние.** На пользователей не влияет. Телеметрия этого периода частично потеряна (сколько — счётчики
`otelcol_exporter_send_failed_*`, `otelcol_exporter_enqueue_failed_*`).
**Найти причину.** `docker compose ps tempo loki prometheus`; `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`;
логи Collector: `docker compose logs otel-collector --tail 50` (через Loki их не смотреть — это тот же конвейер).
**Исправить.** `docker compose start tempo loki prometheus`. Очередь Collector досылается сама (в пределах окна
повторов). Проверить: `otelcol_exporter_queue_size` → 0, свежая трасса находится. Пример: REPORT.md, инцидент 3.
---
## TelemetryCollectorDown
**Что значит.** Prometheus не может забрать метрики Collector (`up{job="otel-collector"} == 0`).
**Влияние.** Приложение работает; SDK каждого процесса держит не больше 2048 спанов и 2048 записей логов.
Потери считаются в самом процессе: пакет, который Collector не принял, — `otel_sdk_exporter_span_exported_total{error_type!=""}`
(обычно `ConnectionError`), переполнение очереди — `otel_sdk_processor_span_processed_total{error_type="queue_full"}`. Метрик приложения за
этот период нет (пропуск на графиках), cumulative-счётчики после восстановления продолжат расти без потери итогов.
**Исправить.** `docker compose start otel-collector`; проверить, что трассы и метрики снова приходят.
+50
View File
@@ -0,0 +1,50 @@
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
class ApiError(Exception):
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers or {}
def bad_request(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
def not_found(message: str) -> ApiError:
return ApiError(404, "not_found", message)
def conflict(message: str) -> ApiError:
return ApiError(409, "conflict", message)
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
return ApiError(503, "unavailable", message)
def payload_too_large(message: str) -> ApiError:
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(message: str) -> ApiError:
return ApiError(415, "unsupported_media_type", message)
def rate_limited(retry_after: int, message: str = "Too many attempts, retry later") -> ApiError:
"""lab7: 429 + Retry-After (секунды до конца окна общего счётчика в Redis)."""
return ApiError(429, "rate_limited", message, {"Retry-After": str(max(1, int(retry_after)))})
def busy(retry_after: int = 1) -> ApiError:
"""lab7: бюджет одновременных Argon2id-хешей реплики исчерпан — 503 + Retry-After, без очереди."""
return ApiError(503, "unavailable", "Server is busy, retry later", {"Retry-After": str(max(1, int(retry_after)))})
+60
View File
@@ -0,0 +1,60 @@
{
"rows": [
{
"m_kib": 19456,
"t": 2,
"p": 1,
"hash_ms": 12.2,
"verify_ms": 12.2,
"max_logins_per_s_per_replica": 163.9
},
{
"m_kib": 19456,
"t": 3,
"p": 1,
"hash_ms": 18.3,
"verify_ms": 18.5,
"max_logins_per_s_per_replica": 108.1
},
{
"m_kib": 32768,
"t": 2,
"p": 1,
"hash_ms": 22.6,
"verify_ms": 22.2,
"max_logins_per_s_per_replica": 90.1
},
{
"m_kib": 47104,
"t": 2,
"p": 1,
"hash_ms": 34.2,
"verify_ms": 34.8,
"max_logins_per_s_per_replica": 57.5
},
{
"m_kib": 65536,
"t": 2,
"p": 1,
"hash_ms": 49.8,
"verify_ms": 50.4,
"max_logins_per_s_per_replica": 39.7
},
{
"m_kib": 65536,
"t": 3,
"p": 1,
"hash_ms": 75.3,
"verify_ms": 75.2,
"max_logins_per_s_per_replica": 26.6
}
],
"maxrss_mib": 96.1,
"current": [
32768,
2,
1
],
"max_concurrency": 2,
"queue_s": 2
}
+14
View File
@@ -0,0 +1,14 @@
# argon2-bench: lab7: Argon2id cost inside the api container for candidate parameters
# date: 2026-11-07 19:08:27, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/argon2_bench.py
# exit code: 0, 3.9 s
m (KiB) t p hash ms verify ms logins/s per replica at concurrency 2 RAM for hashing
19456 2 1 12.2 12.2 163.9 38 MiB
19456 3 1 18.3 18.5 108.1 38 MiB
32768 2 1 22.6 22.2 90.1 64 MiB <- current
47104 2 1 34.2 34.8 57.5 92 MiB
65536 2 1 49.8 50.4 39.7 128 MiB
65536 3 1 75.3 75.2 26.6 128 MiB
process max RSS during the bench: 96.1 MiB
saved evidence/argon2-bench.json
+96
View File
@@ -0,0 +1,96 @@
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
# date: 2026-11-07 19:40:25, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_attachments.py
# exit code: 0, 12.8 s
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
exactly 10485760 bytes (10 MiB) -> 202, then ready
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
PASS: the 10 MiB limit is exact
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
PASS: oversized streamed body is cut off, the API keeps working
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 54186d3d
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
PASS: pixel limit is checked before decompression
PNG cut in half (55908 bytes, image/png) -> 415
PNG without IEND (111805 bytes, image/png) -> 415
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
PNG with garbage after IEND (111824 bytes, image/png) -> 415
PNG signature only (8 bytes, image/png) -> 415
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
JPEG cut in half (746 bytes, image/jpeg) -> 415
JPEG header only (20 bytes, image/jpeg) -> 415
random bytes (1600 bytes, image/png) -> 415
plain text (28 bytes, image/jpeg) -> 415
empty body -> 400
PASS: damaged and foreign files are rejected with 415, never 500
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
empty PLTE (79 bytes, valid CRCs) -> 415
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
palette image without PLTE (67 bytes, valid CRCs) -> 415
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
PLTE after IDAT (85 bytes, valid CRCs) -> 415
two PLTE chunks (103 bytes, valid CRCs) -> 415
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
tRNS before PLTE (98 bytes, valid CRCs) -> 415
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
IEND with data (70 bytes, valid CRCs) -> 415
unknown critical chunk (81 bytes, valid CRCs) -> 415
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
gAMA after IDAT (85 bytes, valid CRCs) -> 415
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
private chunk after IDAT (181 bytes, valid CRCs) -> 415
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
palette depth 2 with 4 entries -> 201, downloaded byte for byte
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
PNG bytes sent as image/jpeg -> 415
JPEG bytes sent as image/png -> 415
GIF bytes sent as image/png -> 415
PNG sent as image/gif -> 415
PNG sent as text/html -> 415
PNG sent as application/octet-stream -> 415
PNG sent as multipart/form-data -> 415
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
PASS: the type is taken from the content; a declared type that does not match is refused
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
no filename -> 400
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
PASS: filename never becomes a path; dangerous names are refused
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
upload into someone else's / unknown chat -> 404, without token -> 401
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
PASS: only chat members can see an attachment
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
attachment of another chat (author is a member of both) -> 404
own + foreign attachment -> 404
unknown attachment_id -> 404
outsider referencing an attachment of the chat -> 404
duplicate ids -> 400
11 ids -> 400
bad UUID -> 400
empty text without attachments -> 400
whitespace text without attachments -> 400
PASS: only attachments of the same chat can be linked; rejected requests change nothing
same key + same attachments -> same 201; other attachments -> 409; missing field == []
PASS: Idempotency-Key covers attachment_ids
sent 1.1 MB of a declared 5 MB body and closed the connection
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
PASS: an interrupted upload is dropped
12 passed, 0 failed
+18
View File
@@ -0,0 +1,18 @@
# balancing: balancing, shared sessions/data/files, X-Instance-Id
# date: 2026-11-07 19:43:46, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_balancing.py
# exit code: 0, 3.5 s
20 x GET /users/me with one token -> replicas: {'f9e5ac61cde6': 10, '708b4b92f569': 10}
PASS: one session works on every replica, no second login (sessions are shared in Redis)
message written by 708b4b92f569; read back by ['708b4b92f569', 'f9e5ac61cde6']
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
logout on f9e5ac61cde6; old token then rejected by ['708b4b92f569', 'f9e5ac61cde6']
PASS: logout on one replica revokes the session on all of them
first POST on f9e5ac61cde6 -> 201 b67e1df0-7002-437c-ba25-1ca953b58180; same key on 708b4b92f569 -> 201 b67e1df0-7002-437c-ba25-1ca953b58180
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
file uploaded via f9e5ac61cde6; downloaded byte for byte via ['708b4b92f569', 'f9e5ac61cde6']
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running replica ['708b4b92f569', 'f9e5ac61cde6']; api containers -> aliases: {'0c7893e1043d': '708b4b92f569', 'b7f37f051ba1': 'f9e5ac61cde6'}
PASS: X-Instance-Id is in every response and points to a real container
6 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# check: offline contract check (make check)
# date: 2026-11-07 19:07:13, commit: 20afc2c + uncommitted changes
# command: python.exe scripts/check_contract.py
# exit code: 0, 0.2 s
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+34
View File
@@ -0,0 +1,34 @@
# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning
# date: 2026-11-07 19:12:52, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/check_configs.py
# exit code: 0, 5.7 s
--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0
--- promtool check config (infra/prometheus/prometheus.yml): exit 0
Checking /etc/prometheus/rules/alerts.yml
SUCCESS: 6 rules found
Checking /etc/prometheus/rules/recording.yml
SUCCESS: 10 rules found
--- promtool check rules (recording + alerts): exit 0
Checking /etc/prometheus/rules/alerts.yml
SUCCESS: 6 rules found
Checking /etc/prometheus/rules/recording.yml
SUCCESS: 10 rules found
--- promtool check web-config (lab7: TLS, client certs, allowed SANs): exit 0
/etc/prometheus/web.yml SUCCESS
--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0
SUCCESS
--- loki -verify-config (infra/loki/loki.yaml): exit 0
--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0
--- Grafana 12.4.14: database ok
datasource prometheus (prometheus): OK Successfully queried the Prometheus API.
datasource loki (loki): OK Data source successfully connected.
datasource tempo (tempo): OK Data source is working
dashboard 'Messenger — lab6 observability' in folder 'Messenger': 34 panels, provisioned=True, datasources ['loki', 'prometheus', 'tempo']
PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana provisioned 3 healthy datasources and the dashboard without manual steps
+47
View File
@@ -0,0 +1,47 @@
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
# date: 2026-11-07 19:42:46, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_consistency.py
# exit code: 0, 131.5 s
declared 9 000 000 bytes, sent 211817, closed the connection
declared 9 000 000 bytes, sent 3111817, closed the connection
rows in attachments: 481 -> 481; objects in bucket: 1886 -> 1886
temporary upload files in api containers: {'b7f37f051ba1': 0, '0c7893e1043d': 0} (removed when the request ends)
api containers have no volumes: the original is never stored locally
PASS: an interrupted upload leaves no row, no object and no local file
crash after S3 PUT: row aa892679 state=uploading, object staging/a6d6cacccc714e1683e0cbf58f6c3bb8 exists in S3
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1891, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.698} -> fresh row and object are kept (the upload might still be running)
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1891, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.695} -> nothing deleted
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1890, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.691} -> row and object deleted
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1890, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.599} -> idempotent
the ready attachment of the same chat is untouched
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
object staging/71f7fcf571ca43e18de2aa82aac94d8e put into the bucket without a row in PostgreSQL
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1895, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.689} -> fresh object kept
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1895, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.793} -> reported, kept
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1895, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 2, "seconds": 0.678} -> orphan deleted
ready attachments older than the grace period are untouched (they have rows)
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=3 and keeps the row; download -> 503 (not 500)
PASS: orphans are found and deleted only after the grace period; missing objects are reported
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
upload -> 503 unavailable in 781 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose start s3
after S3 recovery the old file downloads byte for byte
new upload -> 202 (14.7 s after the old file became readable)
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1905, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 3, "seconds": 0.695}
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
--- S3 frozen (no answer, timeouts): docker compose pause s3
upload -> 503 unavailable in 8703 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose unpause s3
after S3 recovery the old file downloads byte for byte
new upload -> 202 (0.3 s after the old file became readable)
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 1914, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 3, "seconds": 0.685}
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
5 passed, 0 failed
+36
View File
@@ -0,0 +1,36 @@
# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels
# date: 2026-11-07 19:14:22, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/check_dashboard.py
# exit code: 0, 2.7 s
PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident)
empty now: CPU контейнеров / лимит
empty now: Collector: потери и отказы
empty now: SDK приложений: экспорт и потери
empty now: alerts.yml: MessengerDbPoolSaturated
empty now: alerts.yml: MessengerHighErrorRate
empty now: alerts.yml: MessengerQueueLagGrowing
empty now: alerts.yml: MessengerWorkerNoConsumers
empty now: alerts.yml: TelemetryCollectorDown
empty now: alerts.yml: TelemetryExportFailing
empty now: Алерты Prometheus (pending/firing)
empty now: Память: RSS процесса и контейнер
LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error"
TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 5 trace(s)
data: http_server_request_duration_seconds_count{job="messenger-api"} 54 series <- HTTP requests/duration (3)
data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3)
data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4)
data: messenger_queue_depth 2 series <- queue depth (4)
data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4)
data: db_client_connection_count 6 series <- DB pool used/idle (4)
data: db_client_connection_pending_requests 3 series <- DB pool waiting (4)
data: messenger_worker_busy 1 series <- worker busy (4)
data: messenger_worker_capacity 1 series <- worker capacity (4)
data: process_cpu_time_seconds_total 16 series <- process CPU (4)
data: process_memory_usage_bytes 8 series <- process RAM (4)
data: otelcol_exporter_queue_size 3 series <- collector queue (4)
labels used by application metrics: __name__, bao_role, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, error_type, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, messenger_rate_limit, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready']
PASS: every dashboard/rule query runs, key signals have data, no forbidden labels
+21
View File
@@ -0,0 +1,21 @@
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
# date: 2026-11-07 19:44:08, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_db.py
# exit code: 0, 11.1 s
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 389 -> 390
PASS: chat row was inserted and then rolled back together with members
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
PASS: the database itself rejects a message from a non-member
stored password: $argon2id$v=19$m=32768,t=2,p=1$... (length 97)
PASS: password is stored as Argon2id hash
unique constraint on users: UNIQUE (username)
PASS: username is unique in the database
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; running the migrate service once more
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; log: Migrations applied: none (schema is up to date)
PASS: migrations are applied once, a repeated run does nothing
api replicas that checked the schema at start: 11; api log lines 'Migrations applied': 0; migrate service: Exited (0) 3 minutes ago
PASS: only the migrate service applies migrations, replicas wait for the schema
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
7 passed, 0 failed
+112
View File
@@ -0,0 +1,112 @@
=== Data volume
messages_total | messages_in_big_chat
----------------+----------------------
141938 | 15000
=== History index
indexdef
----------------------------------------------------------------------------------------
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
QUERY PLAN
------------------------------------------------------------------------------------------------
Limit (actual time=0.007..0.018 rows=51 loops=1)
Buffers: shared hit=54
-> Index Scan using messages_history on messages (actual time=0.007..0.015 rows=51 loops=1)
Index Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
Buffers: shared hit=54
Planning:
Buffers: shared hit=7
Planning Time: 0.066 ms
Execution Time: 0.026 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.027..0.041 rows=51 loops=1)
Buffers: shared hit=4
-> Index Scan using messages_history on messages (actual time=0.026..0.036 rows=51 loops=1)
Index Cond: ((chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'cc573555-8177-419f-a1b0-2b402c9b73d3'::uuid)))
Buffers: shared hit=4
Planning Time: 0.084 ms
Execution Time: 0.098 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.021..0.177 rows=51 loops=1)
Buffers: shared hit=58
-> Index Scan using messages_history on messages m (actual time=0.021..0.174 rows=51 loops=1)
Index Cond: ((chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'cc573555-8177-419f-a1b0-2b402c9b73d3'::uuid)))
Buffers: shared hit=58
SubPlan 1
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
Sort Key: a."position"
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
Recheck Cond: (message_id = m.id)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Index Cond: (message_id = m.id)
Buffers: shared hit=51
Planning:
Buffers: shared hit=63
Planning Time: 0.164 ms
Execution Time: 0.219 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
QUERY PLAN
-------------------------------------------------------------------------------------------------------
Limit (actual time=3.398..3.401 rows=51 loops=1)
Buffers: shared hit=2553
-> Sort (actual time=3.012..3.217 rows=10051 loops=1)
Sort Key: created_at, id
Sort Method: quicksort Memory: 2377kB
Buffers: shared hit=2553
-> Bitmap Heap Scan on messages (actual time=0.608..1.680 rows=15000 loops=1)
Recheck Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
Heap Blocks: exact=2359
Buffers: shared hit=2553
-> Bitmap Index Scan on messages_history (actual time=0.458..0.458 rows=15000 loops=1)
Index Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
Buffers: shared hit=194
Planning Time: 0.036 ms
Execution Time: 3.484 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
QUERY PLAN
-----------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=1.636..1.639 rows=51 loops=1)
Buffers: shared hit=2375
-> Sort (actual time=1.635..1.637 rows=51 loops=1)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 31kB
Buffers: shared hit=2375
-> Bitmap Heap Scan on messages (actual time=1.034..1.289 rows=5000 loops=1)
Recheck Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
Filter: (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'cc573555-8177-419f-a1b0-2b402c9b73d3'::uuid))
Rows Removed by Filter: 10000
Heap Blocks: exact=2359
Buffers: shared hit=2375
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.303..0.303 rows=15000 loops=1)
Index Cond: (chat_id = '1efc622c-6386-4403-a0dc-6d7e17977038'::uuid)
Buffers: shared hit=16
Planning:
Buffers: shared hit=4
Planning Time: 0.099 ms
Execution Time: 1.665 ms
=== ROLLBACK: test data removed, index is back
explain_users_left
--------------------
0
+48
View File
@@ -0,0 +1,48 @@
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
# date: 2026-11-07 19:44:13, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/explain.py
# exit code: 0, 0.8 s
=== Data volume
=== History index
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
Buffers: shared hit=54
-> Index Scan using messages_history on messages (actual time=0.007..0.015 rows=51 loops=1)
Buffers: shared hit=54
Buffers: shared hit=7
Execution Time: 0.026 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
Buffers: shared hit=4
-> Index Scan using messages_history on messages (actual time=0.026..0.036 rows=51 loops=1)
Buffers: shared hit=4
Execution Time: 0.098 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
Buffers: shared hit=58
-> Index Scan using messages_history on messages m (actual time=0.021..0.174 rows=51 loops=1)
Buffers: shared hit=58
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
Buffers: shared hit=63
Execution Time: 0.219 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
Buffers: shared hit=2553
Buffers: shared hit=2553
-> Bitmap Heap Scan on messages (actual time=0.608..1.680 rows=15000 loops=1)
Buffers: shared hit=2553
-> Bitmap Index Scan on messages_history (actual time=0.458..0.458 rows=15000 loops=1)
Buffers: shared hit=194
Execution Time: 3.484 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
Buffers: shared hit=2375
Buffers: shared hit=2375
-> Bitmap Heap Scan on messages (actual time=1.034..1.289 rows=5000 loops=1)
Buffers: shared hit=2375
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.303..0.303 rows=15000 loops=1)
Buffers: shared hit=16
Buffers: shared hit=4
Execution Time: 1.665 ms
=== ROLLBACK: test data removed, index is back
PASS: full plans saved to evidence\explain-history.txt
+33
View File
@@ -0,0 +1,33 @@
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
# date: 2026-11-07 19:45:00, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/run_failover.py
# exit code: 0, 46.3 s
hook: scripts\stop-one.cmd; api replicas: ['0c7893e1043d', 'b7f37f051ba1']
=== run 1: api containers before
SERVICE CONTAINER ID STATUS
api b7f37f051ba1 Up 49 seconds (healthy)
api 0c7893e1043d Up 49 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 5.453}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
SERVICE CONTAINER ID STATUS
api b7f37f051ba1 Up About a minute (healthy)
api 0c7893e1043d Exited (0) 11 seconds ago
the hook stopped 0c7893e1043d (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
restored: docker start 0c7893e1043d -> healthy and back in HAProxy
=== run 2: api containers before
SERVICE CONTAINER ID STATUS
api b7f37f051ba1 Up About a minute (healthy)
api 0c7893e1043d Up 5 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 5.313}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
SERVICE CONTAINER ID STATUS
api b7f37f051ba1 Exited (0) 11 seconds ago
api 0c7893e1043d Up 23 seconds (healthy)
the hook stopped b7f37f051ba1 (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
restored: docker start b7f37f051ba1 -> healthy and back in HAProxy
PASS: public failover test passed 2 times; every api replica was stopped once and restored
+29
View File
@@ -0,0 +1,29 @@
# failures: Redis and PostgreSQL outages via the balancer
# date: 2026-11-07 19:46:03, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_failures.py
# exit code: 0, 32.4 s
stopping redis
/health/live -> 200, /health/ready -> 503 (0 ms)
valid token: GET /users/me -> 503 unavailable (16 ms), session cannot be checked
unknown token -> 503, not 200: nobody is trusted without a check
no token -> 401
login -> 503: there is nowhere to store the session
registration -> 201: it needs only PostgreSQL
starting redis
after start: the old token works, confirmed messages are in place
PASS: Redis outage gives 503 and does not let anyone in without a session check
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
right after the crash: GET /users/me -> 503
starting redis
after start: the session issued before the crash still works (AOF, appendfsync always)
PASS: sessions survive a Redis crash, not only a clean restart
stopping postgres
/health/live -> 200, /health/ready -> 503 (1016 ms)
valid token: users/me, chats, history, new message -> 503 unavailable, no details
no token -> 401, unknown token -> 401: anonymous users are not let in
login -> 503, registration -> 503
starting postgres
after start: the old token works, confirmed messages are in place
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
3 passed, 0 failed
File diff suppressed because it is too large Load Diff
+28
View File
@@ -0,0 +1,28 @@
# hardening: lab7: no privileged/host net/docker.sock, caps dropped, no-new-privileges, read-only, non-root, limits, ports
# date: 2026-11-07 19:07:19, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/check_hardening.py --runtime
# exit code: 0, 0.8 s
EXCEPTION otel-collector network network_mode service:otel-gate — общий namespace с mTLS-шлюзом, Collector слушает только 127.0.0.1
EXCEPTION rabbitmq read_only Erlang/RabbitMQ пишет cookie, схему и plugins-expand в /var/lib/rabbitmq и служебные файлы в /etc/rabbitmq при старте; данные — в volume, корень остав
EXCEPTION telemetry-init cap_add одноразовый chown volumes Loki/Tempo для uid 10001: только CAP_CHOWN, без сети (network_mode none), сразу завершается
EXCEPTION telemetry-init non_root chown требует uid 0 (+CAP_CHOWN)
static: {'PASS': 183, 'EXCEPTION': 4, 'FAIL': 0}
runtime api user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
runtime api user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
runtime dispatcher user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=134217728 pids=256 ports=[]
runtime grafana user=472 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=402653184 pids=256 ports=['3000/tcp']
runtime janitor user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=201326592 pids=256 ports=[]
runtime lb user=haproxy ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=134217728 pids=256 ports=['8404/tcp', '8443/tcp']
runtime loki user=10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=['3100/tcp']
runtime openbao user=100 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=268435456 pids=256 ports=['8200/tcp']
runtime otel-collector user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=402653184 pids=256 ports=[]
runtime otel-gate user=haproxy ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=67108864 pids=256 ports=[]
runtime postgres user=70:70 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
runtime prometheus user=nobody ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=['9090/tcp']
runtime rabbitmq user=rabbitmq ro=False priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=512 ports=[]
runtime redis user=999:1000 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=134217728 pids=256 ports=[]
runtime s3 user=1000:1000 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=[]
runtime tempo user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=536870912 pids=256 ports=['3200/tcp']
runtime worker user=10001:10001 ro=True priv=False capdrop=['ALL'] capadd=None nnp=True mem=1610612736 pids=128 ports=[]
PASS: no privileged/host network/docker.sock; caps dropped; no-new-privileges; read-only roots; non-root; CPU/RAM/PID limits; only 127.0.0.1 ports — exceptions are explained
+22
View File
@@ -0,0 +1,22 @@
# idempotency: Idempotency-Key, parallel same key via replicas
# date: 2026-11-07 19:43:50, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_idempotency.py
# exit code: 0, 4.0 s
first -> 201; same JSON written differently -> 201; plain retry -> 201
PASS: a retry with the same key returns the original 201 and message, no duplicate
20 parallel POSTs with one key: statuses {201: 20}, replicas {'f9e5ac61cde6': 10, '708b4b92f569': 10}, different bodies in answers: 1, messages in history: 1
PASS: parallel requests with one key through different replicas create exactly one message
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
PASS: the first body wins, the same body gets 201, the other body gets 409
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
key used by a 400 and by a 404 first, then valid request -> 201
PASS: rejected requests (400, 404) do not store a result for the key
same key and body by an outsider -> 404; by alice after logout -> 401
PASS: a retry checks the session and chat access again, the saved answer is not leaked
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
PASS: an unconfirmed POST can be safely retried with the same key
8 passed, 0 failed
+39
View File
@@ -0,0 +1,39 @@
{
"scenario": "collector-down",
"date": "2026-11-07 19:27:10",
"outage_s": 90,
"baseline": {
"requests": 796,
"codes": {
"201": 796
},
"success_ratio": 1.0,
"success_rps": 53.1,
"p95_ms_successful": 32.0
},
"during": {
"requests": 5298,
"codes": {
"201": 5298
},
"success_ratio": 1.0,
"success_rps": 53.3,
"p95_ms_successful": 32.0
},
"api_mib_before": 831.6,
"api_mib_max": 866.4,
"sdk_lost_spans_api": {
"processor_queue_full": 0,
"exporter_failed": 19456
},
"sdk_exported_ok_after_restart": 5554,
"after": {
"requests": 1347,
"codes": {
"201": 1347
},
"success_ratio": 1.0,
"success_rps": 53.7,
"p95_ms_successful": 32.0
}
}
+20
View File
@@ -0,0 +1,20 @@
# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted
# date: 2026-11-07 19:29:44, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/incidents.py collector-down
# exit code: 0, 154.3 s
=== collector-down
[ 24.6s] baseline: {'requests': 796, 'codes': {'201': 796}, 'success_ratio': 1.0, 'success_rps': 53.1, 'p95_ms_successful': 32.0}, api containers 831.6 MiB
[ 25.1s] docker compose stop otel-collector
[ 29.1s] {'t': 0.5, 'api_mib': 788.1}
[ 43.1s] {'t': 14.5, 'api_mib': 850.0}
[ 57.1s] {'t': 28.5, 'api_mib': 862.5}
[ 71.1s] {'t': 42.5, 'api_mib': 866.4}
[ 85.7s] {'t': 56.4, 'api_mib': 857.3}
[ 99.7s] {'t': 71.1, 'api_mib': 846.4}
[ 113.7s] {'t': 85.0, 'api_mib': 829.6}
[ 124.1s] docker compose start otel-collector
[ 154.2s] API during collector outage: {'requests': 5298, 'codes': {'201': 5298}, 'success_ratio': 1.0, 'success_rps': 53.3, 'p95_ms_successful': 32.0}; api memory 831.6 -> max 866.4 MiB; SDK lost spans (api): {'processor_queue_full': 0, 'exporter_failed': 19456}; exported OK in the last minute after restart: 5554
[ 154.2s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-collector-down.json
PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start
1 passed, 0 failed
+83
View File
@@ -0,0 +1,83 @@
{
"scenario": "db-slow",
"date": "2026-11-07 19:17:54",
"lock_seconds": 210,
"baseline": {
"requests": 2042,
"codes": {
"201": 2042
},
"success_ratio": 1.0,
"success_rps": 102.1,
"p95_ms_successful": 32.0,
"server_p95_s": 0.0236
},
"firing_after_lock_s": 122.5,
"diagnosis": {
"http_5xx_ratio_1m": 1.0,
"http_p95_messages_s": 7.375,
"pool_used": 8.0,
"pool_wait_p95_s": 0.001,
"status_codes_1m": {
"201": 0.0,
"200": 0.0,
"400": 0.0,
"409": 0.0,
"404": 0.0,
"503": 1.44
},
"reads_during_lock_ms": {
"GET /users/me + GET /chats, avg per request": 15.6,
"max": 31.5
},
"api_log_db_unavailable_by_error_type": {
"QueryCanceled": 184
},
"error_trace": {
"trace_id": "50418f127c336af7f2e4dc0190adf7ea",
"server_ms": 5010.0,
"slowest_db_span": "INSERT",
"slowest_db_ms": 5001.6,
"db_span_status": "ERROR",
"db_share": 0.998,
"tree": [
"- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5010.0 ms) ERROR http.response.status_code=503 error.type=503",
" - [api] redis GET (CLIENT, 1.5 ms)",
" - [api] SELECT (CLIENT, 0.6 ms)",
" - [api] INSERT (CLIENT, 5001.6 ms) ERROR"
]
},
"handler_cpu_contrast": {
"trace_id": "2e774b9d5912ba768f3e9881f3c41e80",
"server_ms": 47.0,
"argon2id_ms": 32.2,
"db_ms_total": 6.9,
"tree": [
"- [api] POST /api/v1/users (SERVER, 47.0 ms) http.response.status_code=201",
" - [api] redis INCR (CLIENT, 6.2 ms)",
" - [api] argon2id hash (INTERNAL, 32.2 ms)",
" - [api] INSERT (CLIENT, 0.7 ms)"
]
}
},
"during_lock": {
"requests": 328,
"codes": {
"503": 320,
"201": 8
},
"success_ratio": 0.0244,
"success_rps": 0.0,
"p95_ms_successful": 2031.0
},
"resolved_after_unlock_s": 35.1,
"after": {
"requests": 3086,
"codes": {
"201": 3086
},
"success_ratio": 1.0,
"success_rps": 102.4,
"p95_ms_successful": 32.0
}
}
+58
View File
@@ -0,0 +1,58 @@
# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved
# date: 2026-11-07 19:22:25, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/incidents.py db-slow
# exit code: 0, 271.0 s
=== db-slow
[ 0.1s] load: 8 threads POST /messages
[ 25.2s] baseline: {'requests': 2042, 'codes': {'201': 2042}, 'success_ratio': 1.0, 'success_rps': 102.1, 'p95_ms_successful': 32.0, 'server_p95_s': 0.0236}
[ 25.2s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait)
[ 25.3s] 5xx ratio=0.0 p95=0.024 s pool used=0.0 waiting=0.0
[ 30.4s] 5xx ratio=0.0 p95=0.024 s pool used=0.0 waiting=0.0
[ 35.5s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
[ 40.6s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
[ 45.7s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
[ 50.7s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
[ 55.8s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
[ 60.9s] 5xx ratio=0.008 p95=0.024 s pool used=8.0 waiting=0.0
[ 66.0s] 5xx ratio=0.008 p95=0.025 s pool used=8.0 waiting=0.0
[ 71.1s] 5xx ratio=0.008 p95=0.025 s pool used=8.0 waiting=0.0
[ 76.3s] 5xx ratio=0.016 p95=6.67 s pool used=8.0 waiting=0.0
[ 81.4s] 5xx ratio=0.016 p95=6.737 s pool used=8.0 waiting=0.0
[ 86.5s] 5xx ratio=0.016 p95=7.375 s pool used=8.0 waiting=0.0
[ 91.6s] 5xx ratio=0.14 p95=7.375 s pool used=8.0 waiting=0.0
[ 96.7s] 5xx ratio=0.14 p95=7.375 s pool used=8.0 waiting=0.0
[ 101.8s] 5xx ratio=0.14 p95=7.375 s pool used=8.0 waiting=0.0
[ 106.9s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 112.0s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 117.1s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 122.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 127.4s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 132.5s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 137.6s] 5xx ratio=1.0 p95=7.375 s pool used=2.0 waiting=0.0
[ 142.7s] 5xx ratio=1.0 p95=7.375 s pool used=2.0 waiting=0.0
[ 147.7s] ALERT MessengerHighErrorRate firing (122.5 s after lock)
[ 152.2s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "400": 0.0, "409": 0.0, "404": 0.0, "503": 1.44}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 15.6, "max": 31.5}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 184}}
[ 152.2s] error trace (DB): {"trace_id": "50418f127c336af7f2e4dc0190adf7ea", "server_ms": 5010.0, "slowest_db_span": "INSERT", "slowest_db_ms": 5001.6, "db_span_status": "ERROR", "db_share": 0.998}
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5010.0 ms) ERROR http.response.status_code=503 error.type=503
- [api] redis GET (CLIENT, 1.5 ms)
- [api] SELECT (CLIENT, 0.6 ms)
- [api] INSERT (CLIENT, 5001.6 ms) ERROR
[ 152.2s] contrast, slow handler (CPU): {"trace_id": "2e774b9d5912ba768f3e9881f3c41e80", "server_ms": 47.0, "argon2id_ms": 32.2, "db_ms_total": 6.9}
- [api] POST /api/v1/users (SERVER, 47.0 ms) http.response.status_code=201
- [api] redis INCR (CLIENT, 6.2 ms)
- [api] argon2id hash (INTERNAL, 32.2 ms)
- [api] INSERT (CLIENT, 0.7 ms)
[ 235.7s] lock released (psql exit 0), 210.5 s after LOCK
[ 235.7s] ... MessengerHighErrorRate: firing (0 s)
[ 240.7s] ... MessengerHighErrorRate: firing (5 s)
[ 245.8s] ... MessengerHighErrorRate: firing (10 s)
[ 250.8s] ... MessengerHighErrorRate: firing (15 s)
[ 255.8s] ... MessengerHighErrorRate: firing (20 s)
[ 260.8s] ... MessengerHighErrorRate: firing (25 s)
[ 265.8s] ... MessengerHighErrorRate: firing (30 s)
[ 270.8s] ALERT MessengerHighErrorRate resolved 35.1 s after unlock
[ 270.8s] load: during lock {'requests': 328, 'codes': {'503': 320, '201': 8}, 'success_ratio': 0.0244, 'success_rps': 0.0, 'p95_ms_successful': 2031.0}; after {'requests': 3086, 'codes': {'201': 3086}, 'success_ratio': 1.0, 'success_rps': 102.4, 'p95_ms_successful': 32.0}
[ 270.9s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-db-slow.json
PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved
1 passed, 0 failed
+273
View File
@@ -0,0 +1,273 @@
{
"scenario": "telemetry-down",
"date": "2026-11-07 19:22:25",
"stopped": [
"tempo",
"loki"
],
"outage_s": 150,
"baseline": {
"requests": 793,
"codes": {
"201": 793
},
"success_ratio": 1.0,
"success_rps": 52.9,
"p95_ms_successful": 32.0
},
"collector_heap_before_mib": 32.3,
"collector_container_before_mib": 76.9,
"during": {
"requests": 8486,
"codes": {
"201": 8486
},
"success_ratio": 1.0,
"success_rps": 52.9,
"p95_ms_successful": 32.0
},
"samples": [
{
"t": 0.0,
"collector_heap_mib": 51.5,
"collector_rss_mib": 237.8,
"collector_container_mib": 76.9,
"queue": {
"otlp_http/loki": 16.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 7.0
},
"alert": "inactive"
},
{
"t": 12.1,
"collector_heap_mib": 60.3,
"collector_rss_mib": 237.8,
"collector_container_mib": 76.9,
"queue": {
"otlp_http/loki": 16.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 7.0
},
"alert": "inactive"
},
{
"t": 24.8,
"collector_heap_mib": 44.3,
"collector_rss_mib": 237.8,
"collector_container_mib": 76.9,
"queue": {
"otlp_http/loki": 23.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 10.0
},
"alert": "pending"
},
{
"t": 36.9,
"collector_heap_mib": 53.4,
"collector_rss_mib": 237.8,
"collector_container_mib": 77.0,
"queue": {
"otlp_http/loki": 31.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 13.0
},
"alert": "pending"
},
{
"t": 49.0,
"collector_heap_mib": 62.1,
"collector_rss_mib": 239.8,
"collector_container_mib": 81.0,
"queue": {
"otlp_http/loki": 38.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 16.0
},
"alert": "pending"
},
{
"t": 61.1,
"collector_heap_mib": 62.1,
"collector_rss_mib": 239.8,
"collector_container_mib": 87.1,
"queue": {
"otlp_http/loki": 46.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 19.0
},
"alert": "pending"
},
{
"t": 73.8,
"collector_heap_mib": 45.9,
"collector_rss_mib": 247.8,
"collector_container_mib": 87.1,
"queue": {
"otlp_http/loki": 46.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 19.0
},
"alert": "pending"
},
{
"t": 85.9,
"collector_heap_mib": 54.5,
"collector_rss_mib": 247.8,
"collector_container_mib": 87.2,
"queue": {
"otlp_http/loki": 53.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 22.0
},
"alert": "firing"
},
{
"t": 98.0,
"collector_heap_mib": 66.1,
"collector_rss_mib": 248.0,
"collector_container_mib": 87.3,
"queue": {
"otlp_http/loki": 59.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 24.0
},
"alert": "firing"
},
{
"t": 110.0,
"collector_heap_mib": 75.7,
"collector_rss_mib": 254.0,
"collector_container_mib": 95.4,
"queue": {
"otlp_http/loki": 66.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 27.0
},
"alert": "firing"
},
{
"t": 122.1,
"collector_heap_mib": 84.7,
"collector_rss_mib": 262.2,
"collector_container_mib": 101.5,
"queue": {
"otlp_http/loki": 74.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 30.0
},
"alert": "firing"
},
{
"t": 134.9,
"collector_heap_mib": 84.7,
"collector_rss_mib": 262.2,
"collector_container_mib": 103.7,
"queue": {
"otlp_http/loki": 81.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 33.0
},
"alert": "firing"
},
{
"t": 147.0,
"collector_heap_mib": 64.8,
"collector_rss_mib": 264.2,
"collector_container_mib": 103.6,
"queue": {
"otlp_http/loki": 81.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 33.0
},
"alert": "firing"
}
],
"collector_heap_max_mib": 84.7,
"collector_rss_max_mib": 264.2,
"collector_container_max_mib": 103.7,
"TelemetryExportFailing_firing_at_s": 85.9,
"queue_drained_after_start_s": 55.3,
"probe_traces": [
{
"sent_at_s": 0.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 12.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 24.8,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 36.9,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 49.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 61.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 73.8,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 85.9,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 98.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 110.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 122.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 134.9,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 147.0,
"http": 201,
"trace_in_tempo": true
}
],
"lost_in_collector": {
"spans": 2048,
"log_records": 213
},
"resolved_after_start_s": 70.6,
"after": {
"requests": 3489,
"codes": {
"201": 3489
},
"success_ratio": 1.0,
"success_rps": 53.2,
"p95_ms_successful": 32.0
}
}
+29
View File
@@ -0,0 +1,29 @@
# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted
# date: 2026-11-07 19:27:09, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/incidents.py telemetry-down
# exit code: 0, 284.7 s
=== telemetry-down
[ 22.2s] baseline: {'requests': 793, 'codes': {'201': 793}, 'success_ratio': 1.0, 'success_rps': 52.9, 'p95_ms_successful': 32.0}, collector heap 32.3 MiB, container 76.9 MiB
[ 53.5s] docker compose stop tempo loki: stopped
[ 55.6s] {'t': 0.0, 'collector_heap_mib': 51.5, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.9, 'queue': {'otlp_http/loki': 16.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 7.0}, 'alert': 'inactive'}
[ 68.3s] {'t': 12.1, 'collector_heap_mib': 60.3, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.9, 'queue': {'otlp_http/loki': 16.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 7.0}, 'alert': 'inactive'}
[ 80.4s] {'t': 24.8, 'collector_heap_mib': 44.3, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.9, 'queue': {'otlp_http/loki': 23.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'pending'}
[ 92.4s] {'t': 36.9, 'collector_heap_mib': 53.4, 'collector_rss_mib': 237.8, 'collector_container_mib': 77.0, 'queue': {'otlp_http/loki': 31.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 13.0}, 'alert': 'pending'}
[ 104.5s] {'t': 49.0, 'collector_heap_mib': 62.1, 'collector_rss_mib': 239.8, 'collector_container_mib': 81.0, 'queue': {'otlp_http/loki': 38.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 16.0}, 'alert': 'pending'}
[ 117.3s] {'t': 61.1, 'collector_heap_mib': 62.1, 'collector_rss_mib': 239.8, 'collector_container_mib': 87.1, 'queue': {'otlp_http/loki': 46.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'}
[ 129.4s] {'t': 73.8, 'collector_heap_mib': 45.9, 'collector_rss_mib': 247.8, 'collector_container_mib': 87.1, 'queue': {'otlp_http/loki': 46.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'}
[ 141.4s] {'t': 85.9, 'collector_heap_mib': 54.5, 'collector_rss_mib': 247.8, 'collector_container_mib': 87.2, 'queue': {'otlp_http/loki': 53.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'firing'}
[ 153.5s] {'t': 98.0, 'collector_heap_mib': 66.1, 'collector_rss_mib': 248.0, 'collector_container_mib': 87.3, 'queue': {'otlp_http/loki': 59.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 24.0}, 'alert': 'firing'}
[ 165.6s] {'t': 110.0, 'collector_heap_mib': 75.7, 'collector_rss_mib': 254.0, 'collector_container_mib': 95.4, 'queue': {'otlp_http/loki': 66.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'}
[ 178.4s] {'t': 122.1, 'collector_heap_mib': 84.7, 'collector_rss_mib': 262.2, 'collector_container_mib': 101.5, 'queue': {'otlp_http/loki': 74.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 30.0}, 'alert': 'firing'}
[ 190.4s] {'t': 134.9, 'collector_heap_mib': 84.7, 'collector_rss_mib': 262.2, 'collector_container_mib': 103.7, 'queue': {'otlp_http/loki': 81.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 33.0}, 'alert': 'firing'}
[ 202.5s] {'t': 147.0, 'collector_heap_mib': 64.8, 'collector_rss_mib': 264.2, 'collector_container_mib': 103.6, 'queue': {'otlp_http/loki': 81.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 33.0}, 'alert': 'firing'}
[ 214.0s] docker compose start tempo loki
[ 269.3s] exporter queues drained 55.3 s after start
[ 284.6s] API during outage: {'requests': 8486, 'codes': {'201': 8486}, 'success_ratio': 1.0, 'success_rps': 52.9, 'p95_ms_successful': 32.0}
[ 284.6s] probe traces: 13/13 reached Tempo (lost: sent at [] s)
[ 284.6s] lost in Collector (send_failed+enqueue_failed): {'spans': 2048, 'log_records': 213}; collector heap 32.3 -> max 84.7 MiB (memory_limiter 200 MiB), container max 103.7 MiB
[ 284.6s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-telemetry-down.json
PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved
1 passed, 0 failed
+49
View File
@@ -0,0 +1,49 @@
{
"scenario": "worker-stop",
"date": "2026-11-07 19:15:03",
"firing_after_stop_s": {
"MessengerWorkerNoConsumers": 94.0,
"MessengerQueueLagGrowing": 124.5
},
"diagnosis": {
"queue_consumers": 0.0,
"broker_queue_depth": 3.0,
"jobs_backlog_queued": 3.0,
"oldest_job_age_s": 115.4,
"outbox_oldest_age_s": 0.0,
"published_ok_last_5m": 12.39609660691289,
"worker_metrics_last_seen_s": 123.0,
"last_worker_log": {
"event": "service.stopped",
"message": "stopped"
},
"upload_trace_span_names": [
"INSERT",
"NOTIFY",
"POST /api/v1/chats/{chat_id}/attachments",
"S3 PutObject",
"SELECT",
"UPDATE",
"create attachments",
"inspect image",
"publish attachments",
"redis GET"
],
"upload_trace_has_consumer_span": false
},
"ready_after_start_s": [
4.9,
5.0,
5.4
],
"resolved_after_start_s": {
"MessengerWorkerNoConsumers": 45.6,
"MessengerQueueLagGrowing": 45.6
},
"after_recovery": {
"jobs_backlog_queued": 0.0,
"oldest_job_age_s": 0.0,
"queue_consumers": 2.0
},
"time_to_ready_p95_5m_s": 27.8
}
+50
View File
@@ -0,0 +1,50 @@
# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved
# date: 2026-11-07 19:17:54, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/incidents.py worker-stop
# exit code: 0, 170.7 s
=== worker-stop
[ 0.2s] baseline: consumers=2.0, oldest job age=0.0 s
[ 2.6s] docker compose stop worker
[ 2.7s] uploaded 3 attachments: 202 queued (API accepts without worker)
[ 2.8s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
[ 7.9s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
[ 13.0s] consumers=0.0 backlog queued=3.0 oldest age=5.1 s, alerts: {}
[ 18.1s] consumers=0.0 backlog queued=3.0 oldest age=5.1 s, alerts: {}
[ 23.1s] consumers=0.0 backlog queued=3.0 oldest age=15.1 s, alerts: {}
[ 28.2s] consumers=0.0 backlog queued=3.0 oldest age=15.1 s, alerts: {}
[ 33.3s] consumers=0.0 backlog queued=3.0 oldest age=25.2 s, alerts: {}
[ 38.4s] consumers=0.0 backlog queued=3.0 oldest age=25.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 43.5s] consumers=0.0 backlog queued=3.0 oldest age=35.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 48.5s] consumers=0.0 backlog queued=3.0 oldest age=35.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 53.7s] consumers=0.0 backlog queued=3.0 oldest age=45.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 58.7s] consumers=0.0 backlog queued=3.0 oldest age=45.2 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 63.8s] consumers=0.0 backlog queued=3.0 oldest age=55.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 68.8s] consumers=0.0 backlog queued=3.0 oldest age=55.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 73.9s] consumers=0.0 backlog queued=3.0 oldest age=65.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 79.0s] consumers=0.0 backlog queued=3.0 oldest age=65.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 84.1s] consumers=0.0 backlog queued=3.0 oldest age=75.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 89.2s] consumers=0.0 backlog queued=3.0 oldest age=75.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 94.2s] ALERT MessengerWorkerNoConsumers firing (94.0 s after stop)
[ 94.2s] consumers=0.0 backlog queued=3.0 oldest age=85.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 99.3s] consumers=0.0 backlog queued=3.0 oldest age=85.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 104.4s] consumers=0.0 backlog queued=3.0 oldest age=95.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 109.5s] consumers=0.0 backlog queued=3.0 oldest age=95.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 114.6s] consumers=0.0 backlog queued=3.0 oldest age=105.4 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 119.7s] consumers=0.0 backlog queued=3.0 oldest age=105.4 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 124.7s] ALERT MessengerQueueLagGrowing firing (124.5 s after stop)
[ 124.8s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 3.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 115.4, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 12.39609660691289, "worker_metrics_last_seen_s": 123.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET"], "upload_trace_has_consumer_span": false}
[ 128.9s] docker compose start worker
[ 130.3s] all attachments ready 5.4 s after worker start; backlog now 3.0
[ 130.3s] ... MessengerWorkerNoConsumers: firing (0 s)
[ 135.3s] ... MessengerWorkerNoConsumers: firing (5 s)
[ 140.3s] ... MessengerWorkerNoConsumers: firing (10 s)
[ 145.4s] ... MessengerWorkerNoConsumers: firing (15 s)
[ 150.4s] ... MessengerWorkerNoConsumers: firing (20 s)
[ 155.4s] ... MessengerWorkerNoConsumers: firing (25 s)
[ 160.4s] ... MessengerWorkerNoConsumers: firing (30 s)
[ 165.5s] ... MessengerWorkerNoConsumers: firing (35 s)
[ 170.5s] alerts resolved: {'MessengerWorkerNoConsumers': 45.6, 'MessengerQueueLagGrowing': 45.6}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0}
[ 170.6s] saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\incident-worker-stop.json
PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved
1 passed, 0 failed
+8
View File
@@ -0,0 +1,8 @@
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
# date: 2026-11-07 19:40:12, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py concurrency
# exit code: 0, 19.2 s
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.6 s after worker start, peak processing 2 of 8 (6 samples)
1 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
# date: 2026-11-07 19:35:22, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
# exit code: 0, 161.4 s
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
staging source still present after the crash (worker died before cleanup); waiting 60 s window
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 18 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
4 passed, 0 failed
+13
View File
@@ -0,0 +1,13 @@
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
# date: 2026-11-07 19:39:53, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_janitor_replay.py
# exit code: 0, 192.2 s
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
waiting 84 s until the leftovers are older than grace (120 s)
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
ops replay -> job c532cc9d-8fd5-4baa-9dcf-cc7331f50086 ready
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 2, deleted 2
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
1 passed, 0 failed
+11
View File
@@ -0,0 +1,11 @@
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
# date: 2026-11-07 19:31:20, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py limits
# exit code: 0, 74.8 s
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
10 MiB + 8 bytes -> 413 before any job is created
imaging.py on 8000x5000 PNG: exit 0, peak RSS 225.8 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s
1 passed, 0 failed
+9
View File
@@ -0,0 +1,9 @@
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
# date: 2026-11-07 19:32:40, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py outbox-crash
# exit code: 0, 20.3 s
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 60ca1ef4-a5fb-48ad-94bd-5f14f683304d ready, attempt 1, no manual action
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
1 passed, 0 failed
+16
View File
@@ -0,0 +1,16 @@
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
# date: 2026-11-07 19:32:20, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py worker-down broker-down
# exit code: 0, 60.4 s
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
worker stopped
upload while worker is down -> 202 queued, attachment 5d7b65bc-2cb9-4b4a-801e-6c30173dd513
message with attachment 201, content 409, other messages 201; job 6133fe0a-b568-4024-8bb7-4d7d32ed0433 published, queue messages/unacked = (1, 0)
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
worker started -> same job 6133fe0a-b568-4024-8bb7-4d7d32ed0433 ready, thumbnail downloadable; staging object deleted
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
rabbitmq stopped
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 3e57742c-4cc4-48b0-9645-0e742fa323d0
rabbitmq started -> dispatcher published the pending row -> ready
2 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
# date: 2026-11-07 19:36:41, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq
# exit code: 0, 78.5 s
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
2 x storage_unavailable -> retries after 1.70 s and 2.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
ops replay -> same job_id 3d109dba-1036-482a-a6fb-8116c7894b6d attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
schema-invalid message and unknown job_id -> reject -> DLQ (4 message(s)); dlq-replay kept them: ['unknown job', 'unknown job']
4 passed, 0 failed
+14
View File
@@ -0,0 +1,14 @@
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
# date: 2026-11-07 19:30:05, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_jobs.py variants
# exit code: 0, 21.0 s
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
1 passed, 0 failed
+26
View File
@@ -0,0 +1,26 @@
# lab7-security: lab7: HTTPS only, headers, outsider ACL, logout, shared login rate limit, size limits, Argon2id, rehash, no credential leaks
# date: 2026-11-07 19:08:02, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_lab7_security.py
# exit code: 0, 17.3 s
port 8080: closed
empty trust store: rejected by certificate verification (unable to get local issuer certificate)
foreign CA: rejected by certificate verification (unable to get local issuer certificate)
wrong host name: rejected by certificate verification (Hostname mismatch, certificate is not valid for 'evil.example'.)
PASS: only HTTPS on 8443; untrusted CA and wrong host name fail certificate verification
HSTS, nosniff, no Server header; X-Instance-Id is an opaque alias (c4ba1764f1a3)
PASS: security headers
outsider: 6 attempts on a foreign chat/attachment -> 404 (existence not revealed); X-User-Id of the victim is ignored
PASS: ACL — outsider reads/writes nothing, cannot attach a foreign file
PASS: logout -> the same token is 401 on every replica (session deleted in shared Redis)
codes [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429, 429, 429]; answered by replicas ['76a44da5ae93', 'c4ba1764f1a3']; Retry-After 60 s
PASS: login rate limit is shared by all replicas (Redis), 429 + Retry-After, before Argon2id
JSON > 64 KiB -> 413 (with Content-Length and chunked); upload Content-Length 50 MB -> 413 at the LB
PASS: request size limits
stored: $argon2id$v=19$m=32768,t=2,p=1$<16-byte salt>$<hash>; two users with the same password have different hashes
PASS: Argon2id with salt and parameters in the stored string
legacy hash m=12288,t=3 -> after successful login m=32768,t=2,p=1; the new hash still verifies
PASS: old password hash is upgraded on successful login
errors do not echo credentials; Redis stores session:<SHA-256(token)> only; DB stores only the hash
PASS: credentials and session tokens are not revealed
9 passed, 0 failed
+24
View File
@@ -0,0 +1,24 @@
{
"container_logs": {
"password": 0,
"token": 0,
"basic_b64": 0
},
"container_logs_bearer_or_basic_header": 0,
"loki": {
"password": 0,
"token": 0,
"basic_b64": 0
},
"tempo": {
"traces_fetched": 4,
"password": 0,
"token": 0,
"basic_b64": 0,
"authorization_attribute": 0
},
"prometheus_labels": {
"token": 0,
"user_id": 0
}
}
+8
View File
@@ -0,0 +1,8 @@
# leak-check: lab7: password/token/Authorization absent in container logs, Loki, Tempo traces, Prometheus labels
# date: 2026-11-07 19:08:23, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/leak_check.py
# exit code: 0, 21.5 s
{"container_logs": {"password": 0, "token": 0, "basic_b64": 0}, "container_logs_bearer_or_basic_header": 0, "loki": {"password": 0, "token": 0, "basic_b64": 0}, "tempo": {"traces_fetched": 4, "password": 0, "token": 0, "basic_b64": 0, "authorization_attribute": 0}, "prometheus_labels": {"token": 0, "user_id": 0}}
searched: container logs, Loki, Tempo, Prometheus labels; traces fetched: 4
PASS: no password, session token or Authorization value in logs or telemetry
+749
View File
@@ -0,0 +1,749 @@
[
{
"service": "api",
"area": "postgres",
"action": "SELECT users",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "postgres",
"action": "INSERT outbox (enqueue)",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "postgres",
"action": "UPDATE users.username",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "api",
"area": "postgres",
"action": "SELECT outbox",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table outbox"
},
{
"service": "api",
"area": "postgres",
"action": "DELETE messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "api",
"area": "postgres",
"action": "DROP TABLE messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "must be owner of table messages"
},
{
"service": "api",
"area": "postgres",
"action": "CREATE TABLE",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for schema public"
},
{
"service": "api",
"area": "postgres",
"action": "ALTER ROLE (escalate)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied to alter role"
},
{
"service": "api",
"area": "s3",
"action": "PUT staging/",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "s3",
"action": "DELETE staging/ (rollback)",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "s3",
"action": "PUT attachments/ (worker's prefix)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "GET staging/ (upload sources)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "LIST bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "DELETE attachments/",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "CREATE bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password', 'redis_password', 's3_access_key_id', 's3_secret_access_key']"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/worker",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/dispatcher",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/janitor",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/api -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
},
{
"service": "api",
"area": "redis",
"action": "GET session:*",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "redis",
"action": "GET other key",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "No permissions to access a key"
},
{
"service": "api",
"area": "redis",
"action": "KEYS *",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "User api has no permissions to run the 'keys' command"
},
{
"service": "api",
"area": "redis",
"action": "FLUSHALL",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "User api has no permissions to run the 'flushall' command"
},
{
"service": "api",
"area": "redis",
"action": "CONFIG GET",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "User api has no permissions to run the 'config|get' command"
},
{
"service": "api",
"area": "redis",
"action": "ACL LIST",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "User api has no permissions to run the 'acl|list' command"
},
{
"service": "api",
"area": "redis",
"action": "SUBSCRIBE / PUBLISH",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "User api has no permissions to run the 'publish' command"
},
{
"service": "worker",
"area": "postgres",
"action": "SELECT attachments",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "worker",
"area": "postgres",
"action": "SELECT users (passwords)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "worker",
"area": "postgres",
"action": "SELECT messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "worker",
"area": "postgres",
"action": "DELETE attachments",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table attachments"
},
{
"service": "worker",
"area": "postgres",
"action": "UPDATE outbox",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table outbox"
},
{
"service": "worker",
"area": "s3",
"action": "PUT attachments/",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "worker",
"area": "s3",
"action": "LIST attachments/",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "worker",
"area": "s3",
"action": "LIST bucket root",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "worker",
"area": "s3",
"action": "PUT outside known prefixes",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "worker",
"area": "s3",
"action": "CREATE bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "worker",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/api",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/dispatcher",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/janitor",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
},
{
"service": "dispatcher",
"area": "postgres",
"action": "SELECT outbox",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "dispatcher",
"area": "postgres",
"action": "SELECT users",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "dispatcher",
"area": "postgres",
"action": "SELECT messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "dispatcher",
"area": "postgres",
"action": "DELETE attachments",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table attachments"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password']"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/api",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/worker",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/janitor",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
},
{
"service": "janitor",
"area": "postgres",
"action": "SELECT attachments",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "janitor",
"area": "postgres",
"action": "SELECT users",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "janitor",
"area": "postgres",
"action": "INSERT messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "janitor",
"area": "postgres",
"action": "DELETE chats",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table chats"
},
{
"service": "janitor",
"area": "s3",
"action": "LIST bucket",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "janitor",
"area": "s3",
"action": "GET object content",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "janitor",
"area": "s3",
"action": "CREATE bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "janitor",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/api",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/worker",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/dispatcher",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
}
]
+115
View File
@@ -0,0 +1,115 @@
# least-privilege: lab7: each service identity vs PostgreSQL grants, S3 prefixes, OpenBao policies, Redis ACL
# date: 2026-11-07 19:07:44, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/test_least_privilege.py
# exit code: 0, 5.5 s
api -> postgres
PASS SELECT users expect allowed observed allowed
PASS INSERT outbox (enqueue) expect allowed observed allowed
PASS UPDATE users.username expect denied observed denied
PASS SELECT outbox expect denied observed denied
PASS DELETE messages expect denied observed denied
PASS DROP TABLE messages expect denied observed denied
PASS CREATE TABLE expect denied observed denied
PASS ALTER ROLE (escalate) expect denied observed denied
api -> s3
PASS PUT staging/ expect allowed observed allowed
PASS DELETE staging/ (rollback) expect allowed observed allowed
PASS PUT attachments/ (worker's prefix) expect denied observed denied
PASS GET staging/ (upload sources) expect denied observed denied
PASS LIST bucket expect denied observed denied
PASS DELETE attachments/ expect denied observed denied
PASS CREATE bucket expect denied observed denied
api -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/worker expect denied observed denied
PASS read messenger/dispatcher expect denied observed denied
PASS read messenger/janitor expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
api -> redis
PASS GET session:* expect allowed observed allowed
PASS GET other key expect denied observed denied
PASS KEYS * expect denied observed denied
PASS FLUSHALL expect denied observed denied
PASS CONFIG GET expect denied observed denied
PASS ACL LIST expect denied observed denied
PASS SUBSCRIBE / PUBLISH expect denied observed denied
worker -> postgres
PASS SELECT attachments expect allowed observed allowed
PASS SELECT users (passwords) expect denied observed denied
PASS SELECT messages expect denied observed denied
PASS DELETE attachments expect denied observed denied
PASS UPDATE outbox expect denied observed denied
worker -> s3
PASS PUT attachments/ expect allowed observed allowed
PASS LIST attachments/ expect allowed observed allowed
PASS LIST bucket root expect denied observed denied
PASS PUT outside known prefixes expect denied observed denied
PASS CREATE bucket expect denied observed denied
worker -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/api expect denied observed denied
PASS read messenger/dispatcher expect denied observed denied
PASS read messenger/janitor expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
dispatcher -> postgres
PASS SELECT outbox expect allowed observed allowed
PASS SELECT users expect denied observed denied
PASS SELECT messages expect denied observed denied
PASS DELETE attachments expect denied observed denied
dispatcher -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/api expect denied observed denied
PASS read messenger/worker expect denied observed denied
PASS read messenger/janitor expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
janitor -> postgres
PASS SELECT attachments expect allowed observed allowed
PASS SELECT users expect denied observed denied
PASS INSERT messages expect denied observed denied
PASS DELETE chats expect denied observed denied
janitor -> s3
PASS LIST bucket expect allowed observed allowed
PASS GET object content expect denied observed denied
PASS CREATE bucket expect denied observed denied
janitor -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/api expect denied observed denied
PASS read messenger/worker expect denied observed denied
PASS read messenger/dispatcher expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
83 checks, 0 failed; saved to evidence/least-privilege.json
PASS: every service identity can do its job and nothing else that was probed
+25
View File
@@ -0,0 +1,25 @@
{
"old_serial": "70ea18ded7f86e5a",
"new_serial": "22a71730433d9d42",
"probe": {
"valid LB identity (CN=lb)": "http_200",
"foreign CA, CN=lb": "connection_closed_after_handshake",
"our CA, expired, CN=lb": "connection_closed_after_handshake",
"previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"
},
"api_events_after_revoke": {
"tls.crl_loaded": 2,
"tls.revoked_connections_closed": 2,
"tls.client_rejected": 4
},
"load": {
"seconds": 23.1,
"codes": {
"me:200": 243,
"send:201": 243,
"login:201": 11
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
+9
View File
@@ -0,0 +1,9 @@
# lifecycle-rotate-leaf: lab7: LB leaf rotated (HAProxy reload), old leaf revoked via CRL, rejected by API
# date: 2026-11-07 19:09:33, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/lifecycle.py rotate-leaf
# exit code: 0, 27.6 s
--- rotate-leaf: клиентский сертификат LB (CN=lb)
{"old_serial": "70ea18ded7f86e5a", "new_serial": "22a71730433d9d42", "probe": {"valid LB identity (CN=lb)": "http_200", "foreign CA, CN=lb": "connection_closed_after_handshake", "our CA, expired, CN=lb": "connection_closed_after_handshake", "previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"}, "api_events_after_revoke": {"tls.crl_loaded": 2, "tls.revoked_connections_closed": 2, "tls.client_rejected": 4}, "load": {"seconds": 23.1, "codes": {"me:200": 243, "send:201": 243, "login:201": 11}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-rotate-leaf.json
+21
View File
@@ -0,0 +1,21 @@
{
"kv_version": [
5,
6
],
"replicas": 2,
"active_passwords_during_overlap": 2,
"all_replicas_picked_up_after_s": 23.9,
"active_password_hashes_after": 1,
"new_password_hash_matches": true,
"load": {
"seconds": 37.6,
"codes": {
"me:200": 431,
"send:201": 431,
"login:201": 18
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
+9
View File
@@ -0,0 +1,9 @@
# lifecycle-rotate-secret: lab7: Redis password of api rotated via OpenBao under load, old one revoked
# date: 2026-11-07 19:09:06, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/lifecycle.py rotate-secret
# exit code: 0, 38.2 s
--- rotate-secret: пароль Redis ACL-пользователя api
{"kv_version": [5, 6], "replicas": 2, "active_passwords_during_overlap": 2, "all_replicas_picked_up_after_s": 23.9, "active_password_hashes_after": 1, "new_password_hash_matches": true, "load": {"seconds": 37.6, "codes": {"me:200": 431, "send:201": 431, "login:201": 18}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-rotate-secret.json
+21
View File
@@ -0,0 +1,21 @@
{
"sealed_seconds": 75.6,
"restarted_replica_health_while_sealed": "starting",
"replicas_answering_while_sealed": 1,
"restarted_replica_ready_after_unseal_s": 5.4,
"api_events": {
"secrets.refresh_failed": 5,
"secrets.waiting": 8,
"secrets.login": 1
},
"load": {
"seconds": 89.2,
"codes": {
"me:200": 1012,
"send:201": 1012,
"login:201": 43
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
+9
View File
@@ -0,0 +1,9 @@
# lifecycle-sealed: lab7: sealed OpenBao: running replicas serve, restarted replica not ready until unseal
# date: 2026-11-07 19:12:47, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/lifecycle.py sealed
# exit code: 0, 90.8 s
--- sealed: OpenBao запечатан
{"sealed_seconds": 75.6, "restarted_replica_health_while_sealed": "starting", "replicas_answering_while_sealed": 1, "restarted_replica_ready_after_unseal_s": 5.4, "api_events": {"secrets.refresh_failed": 5, "secrets.waiting": 8, "secrets.login": 1}, "load": {"seconds": 89.2, "codes": {"me:200": 1012, "send:201": 1012, "login:201": 43}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready (fail closed) until unseal
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-sealed.json
+18
View File
@@ -0,0 +1,18 @@
{
"role_ttl": "20s / max 40s",
"observed_seconds": 100,
"api_events": {
"secrets.refresh_failed": 2,
"secrets.login": 4
},
"load": {
"seconds": 101.6,
"codes": {
"me:200": 1172,
"send:201": 1172,
"login:201": 49
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
+9
View File
@@ -0,0 +1,9 @@
# lifecycle-token-expiry: lab7: OpenBao token TTL 20 s / max 40 s + revoke: services re-login with their cert
# date: 2026-11-07 19:11:16, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/lifecycle.py token-expiry
# exit code: 0, 102.4 s
--- token-expiry: короткий TTL токенов OpenBao роли api
{"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": {"secrets.refresh_failed": 2, "secrets.login": 4}, "load": {"seconds": 101.6, "codes": {"me:200": 1172, "send:201": 1172, "login:201": 49}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected
saved C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\lifecycle-token-expiry.json
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 7,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.016,
"concurrency": 4,
"successful_requests": 6202,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 206.623,
"successful_latency_ms": {
"p50": 16.0,
"p95": 32.0,
"p99": 47.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+22
View File
@@ -0,0 +1,22 @@
# load: public load generator, 30 s, concurrency 4
# date: 2026-11-07 19:45:30, commit: 20afc2c + uncommitted changes
# command: python.exe tests/load.py --base-url https://127.0.0.1:8443 --ca-file C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\secrets\pki\public\ca.crt --duration 30 --concurrency 4 --output C:\Users\Gigabyte\Desktop\lab7\mtusi2026lab7\evidence\load.json
# exit code: 0, 30.5 s
{
"lab": 7,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.016,
"concurrency": 4,
"successful_requests": 6202,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 206.623,
"successful_latency_ms": {
"p50": 16.0,
"p95": 32.0,
"p99": 47.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+52
View File
@@ -0,0 +1,52 @@
{
"baseline": {
"legit": {
"requests": 694,
"codes": {
"200": 347,
"201": 347
},
"p50_ms": 12.3,
"p95_ms": 34.6,
"p99_ms": 35.5
},
"attack": null
},
"under_attack": {
"legit": {
"requests": 186,
"codes": {
"200": 93,
"201": 93
},
"p50_ms": 91.1,
"p95_ms": 194.1,
"p99_ms": 243.6
},
"attack": {
"requests": 1924,
"codes": {
"401": 360,
"429": 1564
},
"rps": 96.2,
"p50_ms": 116.8,
"p95_ms": 480.5
}
},
"seconds": 20,
"attackers": 16,
"burst_one_replica": {
"requests": 200,
"codes": {
"hashed": 75,
"rejected_503": 125
},
"max_parallel_hashes": 2,
"limit": 2,
"queue_s": 2,
"seconds": 2.11,
"slowest_rejection_s": 2.05,
"maxrss_growth_mib": 74.7
}
}
+13
View File
@@ -0,0 +1,13 @@
# login-dos: lab7: wrong-password login flood: Argon2id budget per replica, 503/429, latency of legit requests
# date: 2026-11-07 19:48:14, commit: 20afc2c + uncommitted changes
# command: python.exe student-tests/login_dos.py
# exit code: 0, 47.5 s
baseline 20 s ...
{"legit": {"requests": 694, "codes": {"200": 347, "201": 347}, "p50_ms": 12.3, "p95_ms": 34.6, "p99_ms": 35.5}, "attack": null}
attack: 16 threads of wrong-password logins over 40 usernames ...
{"legit": {"requests": 186, "codes": {"200": 93, "201": 93}, "p50_ms": 91.1, "p95_ms": 194.1, "p99_ms": 243.6}, "attack": {"requests": 1924, "codes": {"401": 360, "429": 1564}, "rps": 96.2, "p50_ms": 116.8, "p95_ms": 480.5}}
burst of 200 simultaneous password checks in one api process: {"requests": 200, "codes": {"hashed": 75, "rejected_503": 125}, "max_parallel_hashes": 2, "limit": 2, "queue_s": 2, "seconds": 2.11, "slowest_rejection_s": 2.05, "maxrss_growth_mib": 74.7}
attack: 96.2 login attempts/s; 401 (hash computed) 360, 503 busy (rejected before hashing) 0, 429 rate limited 1564
legit p95: 34.6 ms -> 194.1 ms; legit errors: {}
saved evidence/login-dos.json
+29
View File
@@ -0,0 +1,29 @@
# Прогон в нативной песочнице (без Docker)
Решение собиралось в среде, где Docker Hub недоступен. Чтобы проверить код и конфигурации до Docker-прогона,
тот же стенд запускался процессами на одной машине (Linux x86_64, 2 vCPU, 8 GiB на всё сразу): каждый сервис —
на своём loopback-адресе с именем из compose (`/etc/hosts`: postgres, redis, s3, rabbitmq, openbao, otel-collector,
prometheus), конфиги — файлы из `infra/` с подстановкой путей `/run/tls*` → `secrets/pki/...`; команды `docker` /
`docker compose`, которые вызывают тесты и hooks, выполнялись подменой, управляющей процессами (stop/start/
restart/kill/pause, exec в окружении сервиса, политика `restart: unless-stopped`).
Стенд поднят с нуля шагами оператора из REPORT.md: `pki.py init` → `init_secrets.py` → OpenBao → `bao.py init`,
`unseal`, `configure` → остальные сервисы. Затем `python student-tests/collect_evidence.py <шаги>` с
`EVIDENCE_DIR=evidence/native-sandbox` — результаты в этой папке, сводка — `summary.md`.
| Компонент | В песочнице | В compose.yaml |
| --- | --- | --- |
| Приложение (api ×2, worker, dispatcher, janitor, migrate, s3-init) | тот же код, Python 3.11, `requirements.txt` | образ из `Dockerfile` (python:3.11.13-slim) |
| PostgreSQL | 16.15 (TLS, `pg_hba.conf` из infra) | 17.11-alpine |
| Redis | 7.0.15 (`redis.conf` из infra, TLS + ACL) | 7.4.11-alpine |
| SeaweedFS | 4.52 (те же флаги и `security.toml`) | 4.52 |
| RabbitMQ | 4.4.0 (`rabbitmq.conf` из infra, EXTERNAL) | 4.1.4-alpine |
| OpenBao | **HashiCorp Vault 2.1.2** — API-совместимая замена (sys/init, unseal, cert auth, KV v2, policies — те же пути и поля) | openbao/openbao:2.6.2 |
| HAProxy (lb, otel-gate) | 2.8.30 (конфиги из infra; вместо DNS Docker — две статические реплики) | 3.2.25-alpine |
| OTel Collector | заглушка OTLP/HTTP на 127.0.0.1:4319 за настоящим otel-gate (сохраняет пакеты для `leak_check.py`) | otelcol-contrib 0.164.0 |
| Prometheus | 2.45 — только `web.yml` (TLS, `client_allowed_sans`) | v3.15.1 |
| Loki, Tempo, Grafana | не запускались (нет бинарников) — их TLS проверяется только в Docker | 3.7.9, 3.1.1, 12.4.14 |
Не выполнялись в песочнице (нужен Docker или Loki/Tempo/Grafana): `stack`, `hardening --runtime` (статическая
часть — да), `configs`, `telemetry*`, `dashboard`, `incident-*`, `upload-memory`, `s3-security`
(`test_no_open_ports_or_console`), `rolling-update`, `failover-metrics`, `test_expiry.py`.
+60
View File
@@ -0,0 +1,60 @@
{
"rows": [
{
"m_kib": 19456,
"t": 2,
"p": 1,
"hash_ms": 23.4,
"verify_ms": 23.6,
"max_logins_per_s_per_replica": 84.7
},
{
"m_kib": 19456,
"t": 3,
"p": 1,
"hash_ms": 42.2,
"verify_ms": 42.1,
"max_logins_per_s_per_replica": 47.5
},
{
"m_kib": 32768,
"t": 2,
"p": 1,
"hash_ms": 51.2,
"verify_ms": 49.9,
"max_logins_per_s_per_replica": 40.1
},
{
"m_kib": 47104,
"t": 2,
"p": 1,
"hash_ms": 63.9,
"verify_ms": 62.1,
"max_logins_per_s_per_replica": 32.2
},
{
"m_kib": 65536,
"t": 2,
"p": 1,
"hash_ms": 115.8,
"verify_ms": 115.8,
"max_logins_per_s_per_replica": 17.3
},
{
"m_kib": 65536,
"t": 3,
"p": 1,
"hash_ms": 195.8,
"verify_ms": 187.2,
"max_logins_per_s_per_replica": 10.7
}
],
"maxrss_mib": 136.8,
"current": [
32768,
2,
1
],
"max_concurrency": 2,
"queue_s": 2
}
+14
View File
@@ -0,0 +1,14 @@
# argon2-bench: lab7: Argon2id cost inside the api container for candidate parameters
# date: 2026-11-07 15:08:10, commit: 5bfc53a + uncommitted changes
# command: python student-tests/argon2_bench.py
# exit code: 0, 7.5 s
m (KiB) t p hash ms verify ms logins/s per replica at concurrency 2 RAM for hashing
19456 2 1 23.4 23.6 84.7 38 MiB
19456 3 1 42.2 42.1 47.5 38 MiB
32768 2 1 51.2 49.9 40.1 64 MiB <- current
47104 2 1 63.9 62.1 32.2 92 MiB
65536 2 1 115.8 115.8 17.3 128 MiB
65536 3 1 195.8 187.2 10.7 128 MiB
process max RSS during the bench: 136.8 MiB
saved evidence/argon2-bench.json
+96
View File
@@ -0,0 +1,96 @@
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
# date: 2026-11-07 15:20:02, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_attachments.py
# exit code: 0, 14.3 s
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
exactly 10485760 bytes (10 MiB) -> 202, then ready
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
PASS: the 10 MiB limit is exact
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
PASS: oversized streamed body is cut off, the API keeps working
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 889591f7
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
PASS: pixel limit is checked before decompression
PNG cut in half (55908 bytes, image/png) -> 415
PNG without IEND (111805 bytes, image/png) -> 415
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
PNG with garbage after IEND (111824 bytes, image/png) -> 415
PNG signature only (8 bytes, image/png) -> 415
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
JPEG cut in half (746 bytes, image/jpeg) -> 415
JPEG header only (20 bytes, image/jpeg) -> 415
random bytes (1600 bytes, image/png) -> 415
plain text (28 bytes, image/jpeg) -> 415
empty body -> 400
PASS: damaged and foreign files are rejected with 415, never 500
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
empty PLTE (79 bytes, valid CRCs) -> 415
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
palette image without PLTE (67 bytes, valid CRCs) -> 415
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
PLTE after IDAT (85 bytes, valid CRCs) -> 415
two PLTE chunks (103 bytes, valid CRCs) -> 415
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
tRNS before PLTE (98 bytes, valid CRCs) -> 415
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
IEND with data (70 bytes, valid CRCs) -> 415
unknown critical chunk (81 bytes, valid CRCs) -> 415
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
gAMA after IDAT (85 bytes, valid CRCs) -> 415
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
private chunk after IDAT (181 bytes, valid CRCs) -> 415
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
palette depth 2 with 4 entries -> 201, downloaded byte for byte
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
PNG bytes sent as image/jpeg -> 415
JPEG bytes sent as image/png -> 415
GIF bytes sent as image/png -> 415
PNG sent as image/gif -> 415
PNG sent as text/html -> 415
PNG sent as application/octet-stream -> 415
PNG sent as multipart/form-data -> 415
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
PASS: the type is taken from the content; a declared type that does not match is refused
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
no filename -> 400
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
PASS: filename never becomes a path; dangerous names are refused
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
upload into someone else's / unknown chat -> 404, without token -> 401
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
PASS: only chat members can see an attachment
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
attachment of another chat (author is a member of both) -> 404
own + foreign attachment -> 404
unknown attachment_id -> 404
outsider referencing an attachment of the chat -> 404
duplicate ids -> 400
11 ids -> 400
bad UUID -> 400
empty text without attachments -> 400
whitespace text without attachments -> 400
PASS: only attachments of the same chat can be linked; rejected requests change nothing
same key + same attachments -> same 201; other attachments -> 409; missing field == []
PASS: Idempotency-Key covers attachment_ids
sent 1.1 MB of a declared 5 MB body and closed the connection
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
PASS: an interrupted upload is dropped
12 passed, 0 failed
+18
View File
@@ -0,0 +1,18 @@
# balancing: balancing, shared sessions/data/files, X-Instance-Id
# date: 2026-11-07 15:21:37, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_balancing.py
# exit code: 0, 2.8 s
20 x GET /users/me with one token -> replicas: {'4ec098d7198a': 10, '9b3c6b6625a9': 10}
PASS: one session works on every replica, no second login (sessions are shared in Redis)
message written by 9b3c6b6625a9; read back by ['4ec098d7198a', '9b3c6b6625a9']
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
logout on 4ec098d7198a; old token then rejected by ['4ec098d7198a', '9b3c6b6625a9']
PASS: logout on one replica revokes the session on all of them
first POST on 4ec098d7198a -> 201 34d01e03-4930-4859-b912-60d58ae2bca9; same key on 9b3c6b6625a9 -> 201 34d01e03-4930-4859-b912-60d58ae2bca9
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
file uploaded via 4ec098d7198a; downloaded byte for byte via ['4ec098d7198a', '9b3c6b6625a9']
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running replica ['4ec098d7198a', '9b3c6b6625a9']; api containers -> aliases: {'api-1': '4ec098d7198a', 'api-2': '9b3c6b6625a9'}
PASS: X-Instance-Id is in every response and points to a real container
6 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# check: offline contract check (make check)
# date: 2026-11-07 15:07:32, commit: 5bfc53a + uncommitted changes
# command: python scripts/check_contract.py
# exit code: 0, 0.1 s
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+47
View File
@@ -0,0 +1,47 @@
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
# date: 2026-11-07 15:34:26, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_consistency.py
# exit code: 0, 95.1 s
declared 9 000 000 bytes, sent 211817, closed the connection
declared 9 000 000 bytes, sent 3111817, closed the connection
rows in attachments: 69 -> 69; objects in bucket: 264 -> 264
temporary upload files in api containers: {'api-1': 0, 'api-2': 0} (removed when the request ends)
api containers have no volumes: the original is never stored locally
PASS: an interrupted upload leaves no row, no object and no local file
crash after S3 PUT: row bcd86485 state=uploading, object staging/dcc998514ccb4f5385947581f4c192cc exists in S3
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 269, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.192} -> fresh row and object are kept (the upload might still be running)
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 269, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.207} -> nothing deleted
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.181} -> row and object deleted
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.185} -> idempotent
the ready attachment of the same chat is untouched
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
object staging/f07a7443c23b46d3ad398824008ba3b6 put into the bucket without a row in PostgreSQL
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 273, "orphan_objects": 5, "orphan_objects_deleted": 5, "missing_objects": 1, "seconds": 0.208} -> fresh object kept
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.203} -> reported, kept
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 268, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.198} -> orphan deleted
ready attachments older than the grace period are untouched (they have rows)
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500)
PASS: orphans are found and deleted only after the grace period; missing objects are reported
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
upload -> 503 unavailable in 386 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose start s3
after S3 recovery the old file downloads byte for byte
new upload -> 202 (16.2 s after the old file became readable)
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 1, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 278, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.197}
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
--- S3 frozen (no answer, timeouts): docker compose pause s3
upload -> 503 unavailable in 4064 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose unpause s3
after S3 recovery the old file downloads byte for byte
new upload -> 202 (0.3 s after the old file became readable)
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 287, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.195}
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
5 passed, 0 failed
+21
View File
@@ -0,0 +1,21 @@
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
# date: 2026-11-07 15:21:56, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_db.py
# exit code: 0, 7.0 s
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 62 -> 63
PASS: chat row was inserted and then rolled back together with members
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
PASS: the database itself rejects a message from a non-member
stored password: $argon2id$v=19$m=32768,t=2,p=1$... (length 97)
PASS: password is stored as Argon2id hash
unique constraint on users: UNIQUE (username)
PASS: username is unique in the database
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; running the migrate service once more
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql,006_roles.sql; log: Migrations applied: none (schema is up to date)
PASS: migrations are applied once, a repeated run does nothing
api replicas that checked the schema at start: 5; api log lines 'Migrations applied': 0; migrate service: Exited (0)
PASS: only the migrate service applies migrations, replicas wait for the schema
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
7 passed, 0 failed
@@ -0,0 +1,99 @@
=== Data volume
messages_total | messages_in_big_chat
----------------+----------------------
23017 | 15000
=== History index
indexdef
----------------------------------------------------------------------------------------
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
QUERY PLAN
------------------------------------------------------------------------------------------------
Limit (actual time=0.013..0.125 rows=51 loops=1)
Buffers: shared hit=55
-> Index Scan using messages_history on messages (actual time=0.012..0.119 rows=51 loops=1)
Index Cond: (chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid)
Buffers: shared hit=55
Planning:
Buffers: shared hit=6
Planning Time: 0.119 ms
Execution Time: 0.141 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.011..0.030 rows=51 loops=1)
Buffers: shared hit=6
-> Index Scan using messages_history on messages (actual time=0.010..0.025 rows=51 loops=1)
Index Cond: ((chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 05:46:40+03'::timestamp with time zone, '1e03049f-75da-40e8-9821-2d439e107e51'::uuid)))
Buffers: shared hit=6
Planning Time: 0.124 ms
Execution Time: 0.058 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.036..0.142 rows=51 loops=1)
Buffers: shared hit=60
-> Index Scan using messages_history on messages m (actual time=0.035..0.137 rows=51 loops=1)
Index Cond: ((chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 05:46:40+03'::timestamp with time zone, '1e03049f-75da-40e8-9821-2d439e107e51'::uuid)))
Buffers: shared hit=60
SubPlan 1
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
Sort Key: a."position"
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.001..0.001 rows=0 loops=51)
Recheck Cond: (message_id = m.id)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Index Cond: (message_id = m.id)
Buffers: shared hit=51
Planning:
Buffers: shared hit=63
Planning Time: 0.255 ms
Execution Time: 0.202 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
QUERY PLAN
---------------------------------------------------------------------------------------------------
Limit (actual time=2.823..2.840 rows=51 loops=1)
Buffers: shared hit=311
-> Index Scan using messages_history on messages (actual time=0.020..2.302 rows=10051 loops=1)
Index Cond: (chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid)
Buffers: shared hit=311
Planning Time: 0.082 ms
Execution Time: 2.859 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
QUERY PLAN
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=3.356..3.366 rows=51 loops=1)
Buffers: shared hit=318
-> Sort (actual time=3.354..3.359 rows=51 loops=1)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 31kB
Buffers: shared hit=318
-> Seq Scan on messages (actual time=1.132..2.326 rows=5000 loops=1)
Filter: ((chat_id = '88ae823e-42fe-42cd-b02e-4974b4bcbf0f'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 05:46:40+03'::timestamp with time zone, '1e03049f-75da-40e8-9821-2d439e107e51'::uuid)))
Rows Removed by Filter: 18017
Buffers: shared hit=318
Planning:
Buffers: shared hit=4
Planning Time: 0.103 ms
Execution Time: 3.385 ms
=== ROLLBACK: test data removed, index is back
explain_users_left
--------------------
0
+43
View File
@@ -0,0 +1,43 @@
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
# date: 2026-11-07 15:21:57, commit: 5bfc53a + uncommitted changes
# command: python student-tests/explain.py
# exit code: 0, 0.9 s
=== Data volume
=== History index
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
Buffers: shared hit=55
-> Index Scan using messages_history on messages (actual time=0.012..0.119 rows=51 loops=1)
Buffers: shared hit=55
Buffers: shared hit=6
Execution Time: 0.141 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
Buffers: shared hit=6
-> Index Scan using messages_history on messages (actual time=0.010..0.025 rows=51 loops=1)
Buffers: shared hit=6
Execution Time: 0.058 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
Buffers: shared hit=60
-> Index Scan using messages_history on messages m (actual time=0.035..0.137 rows=51 loops=1)
Buffers: shared hit=60
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.001..0.001 rows=0 loops=51)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
Buffers: shared hit=63
Execution Time: 0.202 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
Buffers: shared hit=311
-> Index Scan using messages_history on messages (actual time=0.020..2.302 rows=10051 loops=1)
Buffers: shared hit=311
Execution Time: 2.859 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
Buffers: shared hit=318
Buffers: shared hit=318
-> Seq Scan on messages (actual time=1.132..2.326 rows=5000 loops=1)
Buffers: shared hit=318
Buffers: shared hit=4
Execution Time: 3.385 ms
=== ROLLBACK: test data removed, index is back
PASS: full plans saved to evidence/explain-history.txt
+29
View File
@@ -0,0 +1,29 @@
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
# date: 2026-11-07 15:59:02, commit: 5bfc53a + uncommitted changes
# command: python student-tests/run_failover.py
# exit code: 0, 40.8 s
hook: scripts/stop-one.sh; api replicas: ['api-1', 'api-2']
=== run 1: api containers before
api-1 api-1 Up (healthy)
api-2 api-2 Up (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.82}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
api-1 api-1 Up (healthy)
api-2 api-2 Exited (0)
the hook stopped api-2 (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
restored: docker start api-2 -> healthy and back in HAProxy
=== run 2: api containers before
api-1 api-1 Up (healthy)
api-2 api-2 Up (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.721}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
api-1 api-1 Exited (0)
api-2 api-2 Up (healthy)
the hook stopped api-1 (the replica behind TARGET_INSTANCE_ID, mapped from its alias); other api containers kept running, postgres/redis/lb were not touched
restored: docker start api-1 -> healthy and back in HAProxy
PASS: public failover test passed 2 times; every api replica was stopped once and restored
+29
View File
@@ -0,0 +1,29 @@
# failures: Redis and PostgreSQL outages via the balancer
# date: 2026-11-07 15:24:59, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_failures.py
# exit code: 0, 33.8 s
stopping redis
/health/live -> 200, /health/ready -> 503 (8 ms)
valid token: GET /users/me -> 503 unavailable (7 ms), session cannot be checked
unknown token -> 503, not 200: nobody is trusted without a check
no token -> 401
login -> 503: there is nowhere to store the session
registration -> 201: it needs only PostgreSQL
starting redis
after start: the old token works, confirmed messages are in place
PASS: Redis outage gives 503 and does not let anyone in without a session check
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
right after the crash: GET /users/me -> 503
starting redis
after start: the session issued before the crash still works (AOF, appendfsync always)
PASS: sessions survive a Redis crash, not only a clean restart
stopping postgres
/health/live -> 200, /health/ready -> 503 (1007 ms)
valid token: users/me, chats, history, new message -> 503 unavailable, no details
no token -> 401, unknown token -> 401: anonymous users are not let in
login -> 503, registration -> 503
starting postgres
after start: the old token works, confirmed messages are in place
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
3 passed, 0 failed
File diff suppressed because it is too large Load Diff
+22
View File
@@ -0,0 +1,22 @@
# idempotency: Idempotency-Key, parallel same key via replicas
# date: 2026-11-07 15:21:41, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_idempotency.py
# exit code: 0, 3.8 s
first -> 201; same JSON written differently -> 201; plain retry -> 201
PASS: a retry with the same key returns the original 201 and message, no duplicate
20 parallel POSTs with one key: statuses {201: 20}, replicas {'9b3c6b6625a9': 10, '4ec098d7198a': 10}, different bodies in answers: 1, messages in history: 1
PASS: parallel requests with one key through different replicas create exactly one message
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
PASS: the first body wins, the same body gets 201, the other body gets 409
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
key used by a 400 and by a 404 first, then valid request -> 201
PASS: rejected requests (400, 404) do not store a result for the key
same key and body by an outsider -> 404; by alice after logout -> 401
PASS: a retry checks the session and chat access again, the saved answer is not leaked
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
PASS: an unconfirmed POST can be safely retried with the same key
8 passed, 0 failed
@@ -0,0 +1,8 @@
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
# date: 2026-11-07 15:19:47, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py concurrency
# exit code: 0, 6.2 s
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 3.0 s after worker start, peak processing 2 of 8 (17 samples)
1 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
# date: 2026-11-07 15:16:04, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
# exit code: 0, 133.0 s
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
staging source still present after the crash (worker died before cleanup); waiting 60 s window
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
4 passed, 0 failed
@@ -0,0 +1,13 @@
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
# date: 2026-11-07 15:19:41, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_janitor_replay.py
# exit code: 0, 172.4 s
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
waiting 89 s until the leftovers are older than grace (120 s)
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
ops replay -> job bdf4dc61-b38c-4cf8-9a5a-0a5bbf7a7947 ready
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 1, deleted 1
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
1 passed, 0 failed
+11
View File
@@ -0,0 +1,11 @@
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
# date: 2026-11-07 15:13:13, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py limits
# exit code: 0, 66.6 s
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
10 MiB + 8 bytes -> 413 before any job is created
imaging.py on 8000x5000 PNG: exit 0, peak RSS 223.8 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s
1 passed, 0 failed
+9
View File
@@ -0,0 +1,9 @@
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
# date: 2026-11-07 15:13:51, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py outbox-crash
# exit code: 0, 6.5 s
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job a39e310d-6e0e-425c-baa6-33db9da3e290 ready, attempt 1, no manual action
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
1 passed, 0 failed
+16
View File
@@ -0,0 +1,16 @@
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
# date: 2026-11-07 15:13:44, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py worker-down broker-down
# exit code: 0, 30.6 s
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
worker stopped
upload while worker is down -> 202 queued, attachment 6f25aa5c-c235-40f8-bf9f-520404807cab
message with attachment 201, content 409, other messages 201; job 182a3066-efb5-453d-8d2a-c906da4ac027 published, queue messages/unacked = (1, 1)
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
worker started -> same job 182a3066-efb5-453d-8d2a-c906da4ac027 ready, thumbnail downloadable; staging object deleted
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
rabbitmq stopped
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 4d076cc6-0e4e-4ec7-ba03-10b557817907
rabbitmq started -> dispatcher published the pending row -> ready
2 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
# date: 2026-11-07 15:16:49, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py retry exhausted-replay permanent dlq
# exit code: 0, 45.1 s
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
2 x storage_unavailable -> retries after 1.20 s and 2.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
ops replay -> same job_id 44cac72b-36f8-411d-9b08-cb90fd27bbb9 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job']
4 passed, 0 failed
+14
View File
@@ -0,0 +1,14 @@
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
# date: 2026-11-07 15:12:07, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_jobs.py variants
# exit code: 0, 9.6 s
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
1 passed, 0 failed
+26
View File
@@ -0,0 +1,26 @@
# lab7-security: lab7: HTTPS only, headers, outsider ACL, logout, shared login rate limit, size limits, Argon2id, rehash, no credential leaks
# date: 2026-11-07 15:07:42, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_lab7_security.py
# exit code: 0, 3.8 s
port 8080: something else answers locally, not the API
empty trust store: rejected by certificate verification (unable to get local issuer certificate)
foreign CA: rejected by certificate verification (unable to get local issuer certificate)
wrong host name: rejected by certificate verification (Hostname mismatch, certificate is not valid for 'evil.example'.)
PASS: only HTTPS on 8443; untrusted CA and wrong host name fail certificate verification
HSTS, nosniff, no Server header; X-Instance-Id is an opaque alias (83d68b5bf960)
PASS: security headers
outsider: 6 attempts on a foreign chat/attachment -> 404 (existence not revealed); X-User-Id of the victim is ignored
PASS: ACL — outsider reads/writes nothing, cannot attach a foreign file
PASS: logout -> the same token is 401 on every replica (session deleted in shared Redis)
codes [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429, 429, 429]; answered by replicas ['83d68b5bf960', 'd7b47c3fc484']; Retry-After 60 s
PASS: login rate limit is shared by all replicas (Redis), 429 + Retry-After, before Argon2id
JSON > 64 KiB -> 413 (with Content-Length and chunked); upload Content-Length 50 MB -> 413 at the LB
PASS: request size limits
stored: $argon2id$v=19$m=32768,t=2,p=1$<16-byte salt>$<hash>; two users with the same password have different hashes
PASS: Argon2id with salt and parameters in the stored string
legacy hash m=12288,t=3 -> after successful login m=32768,t=2,p=1; the new hash still verifies
PASS: old password hash is upgraded on successful login
errors do not echo credentials; Redis stores session:<SHA-256(token)> only; DB stores only the hash
PASS: credentials and session tokens are not revealed
9 passed, 0 failed
+21
View File
@@ -0,0 +1,21 @@
{
"container_logs": {
"password": 0,
"token": 0,
"basic_b64": 0
},
"container_logs_bearer_or_basic_header": 0,
"loki": "unavailable: URLError",
"tempo": {
"traces_fetched": 0,
"password": 0,
"token": 0,
"basic_b64": 0
},
"prometheus_labels": "unavailable: URLError",
"otlp_payloads": {
"password": 0,
"token": 0,
"basic_b64": 0
}
}
+8
View File
@@ -0,0 +1,8 @@
# leak-check: lab7: password/token/Authorization absent in container logs, Loki, Tempo traces, Prometheus labels
# date: 2026-11-07 15:08:02, commit: 5bfc53a + uncommitted changes
# command: python student-tests/leak_check.py
# exit code: 0, 20.6 s
{"container_logs": {"password": 0, "token": 0, "basic_b64": 0}, "container_logs_bearer_or_basic_header": 0, "loki": "unavailable: URLError", "tempo": {"traces_fetched": 0, "password": 0, "token": 0, "basic_b64": 0}, "prometheus_labels": "unavailable: URLError", "otlp_payloads": {"password": 0, "token": 0, "basic_b64": 0}}
searched: container logs, Tempo, raw OTLP payloads (741 KiB); traces fetched: 0
PASS: no password, session token or Authorization value in logs or telemetry
@@ -0,0 +1,749 @@
[
{
"service": "api",
"area": "postgres",
"action": "SELECT users",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "postgres",
"action": "INSERT outbox (enqueue)",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "postgres",
"action": "UPDATE users.username",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "api",
"area": "postgres",
"action": "SELECT outbox",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table outbox"
},
{
"service": "api",
"area": "postgres",
"action": "DELETE messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "api",
"area": "postgres",
"action": "DROP TABLE messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "must be owner of table messages"
},
{
"service": "api",
"area": "postgres",
"action": "CREATE TABLE",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for schema public"
},
{
"service": "api",
"area": "postgres",
"action": "ALTER ROLE (escalate)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied to alter role"
},
{
"service": "api",
"area": "s3",
"action": "PUT staging/",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "s3",
"action": "DELETE staging/ (rollback)",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "s3",
"action": "PUT attachments/ (worker's prefix)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "GET staging/ (upload sources)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "LIST bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "DELETE attachments/",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "s3",
"action": "CREATE bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "api",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password', 'redis_password', 's3_access_key_id', 's3_secret_access_key']"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/worker",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/dispatcher",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/janitor",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/api -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "api",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
},
{
"service": "api",
"area": "redis",
"action": "GET session:*",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "api",
"area": "redis",
"action": "GET other key",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "this user has no permissions to access one of the keys used as arguments"
},
{
"service": "api",
"area": "redis",
"action": "KEYS *",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "this user has no permissions to run the 'keys' command"
},
{
"service": "api",
"area": "redis",
"action": "FLUSHALL",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "this user has no permissions to run the 'flushall' command"
},
{
"service": "api",
"area": "redis",
"action": "CONFIG GET",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "this user has no permissions to run the 'config|get' command"
},
{
"service": "api",
"area": "redis",
"action": "ACL LIST",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "this user has no permissions to run the 'acl|list' command"
},
{
"service": "api",
"area": "redis",
"action": "SUBSCRIBE / PUBLISH",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "this user has no permissions to run the 'publish' command"
},
{
"service": "worker",
"area": "postgres",
"action": "SELECT attachments",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "worker",
"area": "postgres",
"action": "SELECT users (passwords)",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "worker",
"area": "postgres",
"action": "SELECT messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "worker",
"area": "postgres",
"action": "DELETE attachments",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table attachments"
},
{
"service": "worker",
"area": "postgres",
"action": "UPDATE outbox",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table outbox"
},
{
"service": "worker",
"area": "s3",
"action": "PUT attachments/",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "worker",
"area": "s3",
"action": "LIST attachments/",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "worker",
"area": "s3",
"action": "LIST bucket root",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "worker",
"area": "s3",
"action": "PUT outside known prefixes",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "worker",
"area": "s3",
"action": "CREATE bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "worker",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/api",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/dispatcher",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/janitor",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "worker",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
},
{
"service": "dispatcher",
"area": "postgres",
"action": "SELECT outbox",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "dispatcher",
"area": "postgres",
"action": "SELECT users",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "dispatcher",
"area": "postgres",
"action": "SELECT messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "dispatcher",
"area": "postgres",
"action": "DELETE attachments",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table attachments"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password']"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/api",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/worker",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/janitor",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "dispatcher",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
},
{
"service": "janitor",
"area": "postgres",
"action": "SELECT attachments",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "janitor",
"area": "postgres",
"action": "SELECT users",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table users"
},
{
"service": "janitor",
"area": "postgres",
"action": "INSERT messages",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table messages"
},
{
"service": "janitor",
"area": "postgres",
"action": "DELETE chats",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "permission denied for table chats"
},
{
"service": "janitor",
"area": "s3",
"action": "LIST bucket",
"expect": "allowed",
"observed": "allowed",
"pass": true,
"detail": ""
},
{
"service": "janitor",
"area": "s3",
"action": "GET object content",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "janitor",
"area": "s3",
"action": "CREATE bucket",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "AccessDenied"
},
{
"service": "janitor",
"area": "openbao",
"action": "own secret keys (no admin credentials)",
"expect": "ok",
"observed": "ok",
"pass": true,
"detail": "['db_password', 's3_access_key_id', 's3_secret_access_key']"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/api",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/api -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/worker",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/worker -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/dispatcher",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/dispatcher -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/migrate",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/migrate -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "read messenger/s3-init",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/messenger/data/s3-init -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "write own secret",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/messenger/data/janitor -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "write ACL policy",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao PUT /v1/sys/policies/acl/probe-evil -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "list auth methods",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao GET /v1/sys/auth -> HTTP 403"
},
{
"service": "janitor",
"area": "openbao",
"action": "create token",
"expect": "denied",
"observed": "denied",
"pass": true,
"detail": "OpenBao POST /v1/auth/token/create -> HTTP 403"
}
]
+115
View File
@@ -0,0 +1,115 @@
# least-privilege: lab7: each service identity vs PostgreSQL grants, S3 prefixes, OpenBao policies, Redis ACL
# date: 2026-11-07 15:07:38, commit: 5bfc53a + uncommitted changes
# command: python student-tests/test_least_privilege.py
# exit code: 0, 2.7 s
api -> postgres
PASS SELECT users expect allowed observed allowed
PASS INSERT outbox (enqueue) expect allowed observed allowed
PASS UPDATE users.username expect denied observed denied
PASS SELECT outbox expect denied observed denied
PASS DELETE messages expect denied observed denied
PASS DROP TABLE messages expect denied observed denied
PASS CREATE TABLE expect denied observed denied
PASS ALTER ROLE (escalate) expect denied observed denied
api -> s3
PASS PUT staging/ expect allowed observed allowed
PASS DELETE staging/ (rollback) expect allowed observed allowed
PASS PUT attachments/ (worker's prefix) expect denied observed denied
PASS GET staging/ (upload sources) expect denied observed denied
PASS LIST bucket expect denied observed denied
PASS DELETE attachments/ expect denied observed denied
PASS CREATE bucket expect denied observed denied
api -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/worker expect denied observed denied
PASS read messenger/dispatcher expect denied observed denied
PASS read messenger/janitor expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
api -> redis
PASS GET session:* expect allowed observed allowed
PASS GET other key expect denied observed denied
PASS KEYS * expect denied observed denied
PASS FLUSHALL expect denied observed denied
PASS CONFIG GET expect denied observed denied
PASS ACL LIST expect denied observed denied
PASS SUBSCRIBE / PUBLISH expect denied observed denied
worker -> postgres
PASS SELECT attachments expect allowed observed allowed
PASS SELECT users (passwords) expect denied observed denied
PASS SELECT messages expect denied observed denied
PASS DELETE attachments expect denied observed denied
PASS UPDATE outbox expect denied observed denied
worker -> s3
PASS PUT attachments/ expect allowed observed allowed
PASS LIST attachments/ expect allowed observed allowed
PASS LIST bucket root expect denied observed denied
PASS PUT outside known prefixes expect denied observed denied
PASS CREATE bucket expect denied observed denied
worker -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/api expect denied observed denied
PASS read messenger/dispatcher expect denied observed denied
PASS read messenger/janitor expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
dispatcher -> postgres
PASS SELECT outbox expect allowed observed allowed
PASS SELECT users expect denied observed denied
PASS SELECT messages expect denied observed denied
PASS DELETE attachments expect denied observed denied
dispatcher -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/api expect denied observed denied
PASS read messenger/worker expect denied observed denied
PASS read messenger/janitor expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
janitor -> postgres
PASS SELECT attachments expect allowed observed allowed
PASS SELECT users expect denied observed denied
PASS INSERT messages expect denied observed denied
PASS DELETE chats expect denied observed denied
janitor -> s3
PASS LIST bucket expect allowed observed allowed
PASS GET object content expect denied observed denied
PASS CREATE bucket expect denied observed denied
janitor -> openbao
PASS own secret keys (no admin credentials) expect ok observed ok
PASS read messenger/api expect denied observed denied
PASS read messenger/worker expect denied observed denied
PASS read messenger/dispatcher expect denied observed denied
PASS read messenger/migrate expect denied observed denied
PASS read messenger/s3-init expect denied observed denied
PASS write own secret expect denied observed denied
PASS write ACL policy expect denied observed denied
PASS list auth methods expect denied observed denied
PASS create token expect denied observed denied
83 checks, 0 failed; saved to evidence/least-privilege.json
PASS: every service identity can do its job and nothing else that was probed
@@ -0,0 +1,25 @@
{
"old_serial": "7867bb9789172458",
"new_serial": "aed4162e0d6ff182",
"probe": {
"valid LB identity (CN=lb)": "http_200",
"foreign CA, CN=lb": "connection_closed_after_handshake",
"our CA, expired, CN=lb": "connection_closed_after_handshake",
"previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"
},
"api_events_after_revoke": {
"tls.crl_loaded": 2,
"tls.revoked_connections_closed": 2,
"tls.client_rejected": 4
},
"load": {
"seconds": 21.3,
"codes": {
"me:200": 265,
"send:201": 265,
"login:201": 10
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
@@ -0,0 +1,9 @@
# lifecycle-rotate-leaf: lab7: LB leaf rotated (HAProxy reload), old leaf revoked via CRL, rejected by API
# date: 2026-11-07 15:08:49, commit: 5bfc53a + uncommitted changes
# command: python student-tests/lifecycle.py rotate-leaf
# exit code: 0, 21.8 s
--- rotate-leaf: клиентский сертификат LB (CN=lb)
{"old_serial": "7867bb9789172458", "new_serial": "aed4162e0d6ff182", "probe": {"valid LB identity (CN=lb)": "http_200", "foreign CA, CN=lb": "connection_closed_after_handshake", "our CA, expired, CN=lb": "connection_closed_after_handshake", "previous (rotated, revoked) LB leaf": "connection_closed_after_handshake"}, "api_events_after_revoke": {"tls.crl_loaded": 2, "tls.revoked_connections_closed": 2, "tls.client_rejected": 4}, "load": {"seconds": 21.3, "codes": {"me:200": 265, "send:201": 265, "login:201": 10}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-rotate-leaf.json
@@ -0,0 +1,21 @@
{
"kv_version": [
1,
2
],
"replicas": 2,
"active_passwords_during_overlap": 2,
"all_replicas_picked_up_after_s": 4.2,
"active_password_hashes_after": 1,
"new_password_hash_matches": true,
"load": {
"seconds": 17.2,
"codes": {
"me:200": 201,
"send:201": 201,
"login:201": 8
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
@@ -0,0 +1,9 @@
# lifecycle-rotate-secret: lab7: Redis password of api rotated via OpenBao under load, old one revoked
# date: 2026-11-07 15:08:27, commit: 5bfc53a + uncommitted changes
# command: python student-tests/lifecycle.py rotate-secret
# exit code: 0, 17.5 s
--- rotate-secret: пароль Redis ACL-пользователя api
{"kv_version": [1, 2], "replicas": 2, "active_passwords_during_overlap": 2, "all_replicas_picked_up_after_s": 4.2, "active_password_hashes_after": 1, "new_password_hash_matches": true, "load": {"seconds": 17.2, "codes": {"me:200": 201, "send:201": 201, "login:201": 8}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-rotate-secret.json
@@ -0,0 +1,21 @@
{
"sealed_seconds": 73.7,
"restarted_replica_health_while_sealed": "unhealthy",
"replicas_answering_while_sealed": 1,
"restarted_replica_ready_after_unseal_s": 3.2,
"api_events": {
"secrets.refresh_failed": 25,
"secrets.waiting": 8,
"secrets.login": 2
},
"load": {
"seconds": 85.2,
"codes": {
"me:200": 1108,
"send:201": 1108,
"login:201": 41
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
@@ -0,0 +1,9 @@
# lifecycle-sealed: lab7: sealed OpenBao: running replicas serve, restarted replica not ready until unseal
# date: 2026-11-07 15:11:57, commit: 5bfc53a + uncommitted changes
# command: python student-tests/lifecycle.py sealed
# exit code: 0, 85.6 s
--- sealed: OpenBao запечатан
{"sealed_seconds": 73.7, "restarted_replica_health_while_sealed": "unhealthy", "replicas_answering_while_sealed": 1, "restarted_replica_ready_after_unseal_s": 3.2, "api_events": {"secrets.refresh_failed": 25, "secrets.waiting": 8, "secrets.login": 2}, "load": {"seconds": 85.2, "codes": {"me:200": 1108, "send:201": 1108, "login:201": 41}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready (fail closed) until unseal
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-sealed.json
@@ -0,0 +1,19 @@
{
"role_ttl": "20s / max 40s",
"observed_seconds": 100,
"api_events": {
"secrets.refresh_failed": 2,
"secrets.login": 6,
"secrets.renewed": 16
},
"load": {
"seconds": 102.1,
"codes": {
"me:200": 1302,
"send:201": 1302,
"login:201": 49
},
"errors_5xx_or_conn": 0,
"errors": []
}
}
@@ -0,0 +1,9 @@
# lifecycle-token-expiry: lab7: OpenBao token TTL 20 s / max 40 s + revoke: services re-login with their cert
# date: 2026-11-07 15:10:32, commit: 5bfc53a + uncommitted changes
# command: python student-tests/lifecycle.py token-expiry
# exit code: 0, 102.6 s
--- token-expiry: короткий TTL токенов OpenBao роли api
{"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": {"secrets.refresh_failed": 2, "secrets.login": 6, "secrets.renewed": 16}, "load": {"seconds": 102.1, "codes": {"me:200": 1302, "send:201": 1302, "login:201": 49}, "errors_5xx_or_conn": 0, "errors": []}}
PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected
saved /home/claude/work/mtusi2026lab7/evidence/native-sandbox/lifecycle-token-expiry.json
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 7,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.028,
"concurrency": 4,
"successful_requests": 4538,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 151.124,
"successful_latency_ms": {
"p50": 24.062,
"p95": 42.691,
"p99": 84.746
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}

Some files were not shown because too many files have changed in this diff Show More