ЛР 7 — БВТ2303 — Васюкова — на 4 #2
+42
-12
@@ -5,19 +5,42 @@ APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
DATABASE_URL=postgresql://user:password@postgres:5432/high_load_messenger
|
||||
POSTGRES_USER=user
|
||||
POSTGRES_PASSWORD=password
|
||||
POSTGRES_DB=high_load_messenger
|
||||
# Redis. Пароль не задан: Redis доступен только внутри compose-сети (порт наружу не публикуется)
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=https://s3:9000
|
||||
|
||||
# --- MinIO / S3 (lab4) ---
|
||||
# Внутренний endpoint, доступен только из compose-сети.
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
# Администратор MinIO (используется только minio и minio-init).
|
||||
MINIO_ROOT_USER=minioadmin
|
||||
MINIO_ROOT_PASSWORD=minioadmin-secret-change-me
|
||||
# Прикладные S3-credentials (ограниченные права). Используются API и cleanup.
|
||||
S3_APP_KEY_ID=messenger-app
|
||||
S3_APP_SECRET_KEY=messenger-app-secret-change-me
|
||||
# Лимиты (контракт + lab4)
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# lab7: раннее ограничение тела запроса middleware (Content-Length).
|
||||
MAX_REQUEST_BODY_BYTES=10485760
|
||||
# Фоновая сверка S3↔БД (level 4)
|
||||
CLEANUP_INTERVAL_SECONDS=300
|
||||
# Grace period: объекты моложе N секунд не удаляются (защита in-flight uploads).
|
||||
ORPHAN_GRACE_SECONDS=3600
|
||||
|
||||
RABBITMQ_USER=messenger
|
||||
RABBITMQ_PASSWORD=messenger-rabbit-change-me
|
||||
RABBITMQ_URL=amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F
|
||||
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
@@ -25,17 +48,24 @@ JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
|
||||
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
|
||||
GRAFANA_ADMIN_PASSWORD=admin
|
||||
|
||||
# TLS and OpenBao: certs лежат в ./certs (монтируются в /certs), OpenBao слушает 8201.
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
TLS_CA_FILE=/run/secrets/ca.crt
|
||||
TLS_CERT_FILE=/run/secrets/service.crt
|
||||
TLS_KEY_FILE=/run/secrets/service.key
|
||||
TLS_CA_FILE=/certs/ca.crt
|
||||
TLS_CERT_FILE=/certs/api-server.crt
|
||||
TLS_KEY_FILE=/certs/api-server.key
|
||||
BAO_ADDR=https://openbao:8200
|
||||
BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
BAO_CACERT=/certs/ca.crt
|
||||
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
# lab7: ограничение частоты входов (общее для реплик через Redis).
|
||||
LOGIN_WINDOW_SECONDS=60
|
||||
LOGIN_MAX_ATTEMPTS=10
|
||||
@@ -3,6 +3,7 @@
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets/
|
||||
certs/
|
||||
certs/private/
|
||||
*.key
|
||||
*.p12
|
||||
@@ -10,6 +11,7 @@ certs/private/
|
||||
*.jks
|
||||
*.pem
|
||||
*.tfstate*
|
||||
*.dump
|
||||
volumes/
|
||||
data/
|
||||
backups/
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# --- build stage ---
|
||||
FROM python:3.12-slim AS builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
|
||||
|
||||
# --- runtime stage ---
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
useradd --create-home --shell /bin/bash appuser
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY src/ ./src/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONPATH=/app/src
|
||||
|
||||
USER appuser
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=15s --retries=3 \
|
||||
CMD python -c "import ssl,urllib.request; ctx=ssl.create_default_context(cafile='/certs/ca.crt'); urllib.request.urlopen('https://127.0.0.1:8080/health/live', context=ctx).read()"
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||
# lab7: API слушает TLS с обязательным клиентским сертификатом (LB->API mTLS).
|
||||
# ssl_cert_reqs=2 = CERT_REQUIRED.
|
||||
CMD ["sh", "-c", "python -m uvicorn app.main:app --host 0.0.0.0 --port 8080 --app-dir src --ssl-ca-certs /certs/ca.crt --ssl-certfile /certs/api-server.crt --ssl-keyfile /certs/api-server.key --ssl-cert-reqs 2"]
|
||||
@@ -4,10 +4,10 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-security
|
||||
.PHONY: help check validate test test-student load test-persistence test-failover test-security
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; test-student: own tests; load: synthetic writes"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -18,6 +18,9 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-student:
|
||||
$(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)"
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,37 +1,333 @@
|
||||
# Отчёт — лабораторная 7
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Васюкова Наталья, БВТ2303
|
||||
- Целевая оценка: 4
|
||||
- Commit/PR: solution/BVT2303-Vasyukova
|
||||
- Использованные источники, библиотеки и помощь ИИ:
|
||||
- Контракт: contracts/openapi.json, contracts/README.md, contracts/observability.md, contracts/trust-matrix.md
|
||||
- Инструкции: README.md, COURSE.md, CONTRIBUTING.md
|
||||
- Библиотеки: fastapi==0.115.0, uvicorn[standard]==0.32.0, psycopg[binary]==3.2.3, psycopg_pool==3.2.3, redis==5.2.0, boto3==1.35.74, Pillow==11.0.0, pika==1.3.2, argon2-cffi==23.1.0, opentelemetry-api==1.31.0, opentelemetry-sdk==1.31.0, opentelemetry-exporter-otlp-proto-http==1.31.0, opentelemetry-instrumentation-fastapi==0.52b0, opentelemetry-instrumentation-requests==0.52b0
|
||||
- ИИ: DeepSeek — помощь в структурировании кода и отчёта. Весь код проверен, отредактирован и запущен разработчиком.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Версии runtime/образов:
|
||||
- Python: 3.12 (образ `python:3.12-slim`)
|
||||
- FastAPI 0.115.0, uvicorn[standard] 0.32.0, psycopg[binary] 3.2.3, psycopg_pool 3.2.3, redis 5.2.0, boto3 1.35.74, Pillow 11.0.0, pika 1.3.2, argon2-cffi 23.1.0
|
||||
- OTel SDK 1.31.0, FastAPI instrumentation 0.52b0
|
||||
- PostgreSQL 18.6-alpine, Redis 7.4.1-alpine, nginx 1.27.3-alpine
|
||||
- RabbitMQ 3.13.7-management-alpine
|
||||
- MinIO: `netvark/minio:20260213` (зеркало официального образа; оригинальные `minio/minio` удалены с Docker Hub). CLI `mc`: `netvark/mc:20260920`.
|
||||
- OpenBao: `openbao/openbao:2.0.2`
|
||||
- OTel Collector: `otel/opentelemetry-collector-contrib:0.108.0`
|
||||
- Prometheus: `prom/prometheus:v2.54.1`, Loki: `grafana/loki:3.1.1`, Tempo: `grafana/tempo:2.5.0`, Grafana: `grafana/grafana:11.2.0`
|
||||
- Docker Desktop (Windows)
|
||||
|
||||
### Запуск стека
|
||||
|
||||
1. Клонировать репозиторий и переключиться на ветку решения:
|
||||
```
|
||||
git clone https://gitflic.ru/project/vasyukova-nat/mtusi2026lab5.git
|
||||
git checkout solution/BVT2303-Vasyukova
|
||||
```
|
||||
2. Создать `.env` из шаблона:
|
||||
```
|
||||
copy .env.example .env
|
||||
```
|
||||
3. Собрать и поднять (LB + 2 реплики API + PostgreSQL + Redis + MinIO + minio-init + rabbitmq + worker + outbox + s3_cleanup + OTel Collector + Prometheus + Loki + Tempo + Grafana + OpenBao + openbao-init + security_init):
|
||||
```
|
||||
docker compose --env-file .env up -d --build
|
||||
```
|
||||
4. Проверить готовность (внешний HTTPS через LB):
|
||||
```
|
||||
curl.exe -s --cacert .\certs\ca.crt https://localhost:8443/health/ready
|
||||
```
|
||||
Ожидается: `{"status":"ok"}`.
|
||||
5. Grafana доступна на `http://localhost:3000` (admin/admin). Дашборд `messenger-lab6` и datasources импортируются автоматически.
|
||||
6. OpenBao доступен только внутри compose-сети по `https://openbao:8200` (наружу порт не публикуется). Bootstrap выполняется автоматически сервисами `security_init` (генерация CA и сертификатов) и `openbao-init` (init/unseal, политики, AppRole).
|
||||
|
||||
Все проверки ниже выполнялись с `PYTHONUTF8=1` (Windows по умолчанию использует cp1251 и падает на чтении UTF-8 JSON):
|
||||
```
|
||||
$env:PYTHONUTF8="1"
|
||||
```
|
||||
|
||||
### Проверки
|
||||
|
||||
`make` в среде автора недоступен (Windows без GNU Make), поэтому использованы эквивалентные прямые вызовы скриптов. Для Linux/CI рядом указаны команды `make`.
|
||||
|
||||
| Makefile-команда | Прямой вызов |
|
||||
| --- | --- |
|
||||
| `make check` | `python scripts/check_contract.py` |
|
||||
| `make validate` | `python scripts/validate_openapi.py` |
|
||||
| `make test` | `python tests/smoke.py --base-url https://localhost:8443 --ca-file certs/ca.crt` |
|
||||
| `make test-security` | `python tests/security.py --base-url https://localhost:8443 --ca-file ./certs/ca.crt` |
|
||||
| `make test-persistence` | `python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url https://localhost:8443 --ca-file ./certs/ca.crt` |
|
||||
| `make test-failover` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url https://localhost:8443 --ca-file ./certs/ca.crt` |
|
||||
| `make test-student` | `python student-tests/lab7_test_level4.py --base-url https://localhost:8443 --ca-file ./certs/ca.crt` |
|
||||
|
||||
### Остановка и удаление учебных данных
|
||||
|
||||
```
|
||||
docker compose stop # данные PostgreSQL, Redis, MinIO, RabbitMQ, Prometheus, Loki, Tempo, Grafana, OpenBao сохраняются (volumes)
|
||||
docker compose down # volumes сохраняются
|
||||
docker compose down -v # ТОЛЬКО если осознанно стираем данные
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Схема процессов и хранилищ:
|
||||
|
||||
```
|
||||
┌─ api-1 (uvicorn:8080, mTLS) ┐
|
||||
HTTPS-клиент → lb:8443 ┤ │
|
||||
(nginx, TLS) └─ api-2 (uvicorn:8080, mTLS)├→ PostgreSQL 18.6 (TLS verify-full)
|
||||
├→ Redis 7.4.1 (сессии)
|
||||
├→ MinIO (S3, private bucket)
|
||||
└→ RabbitMQ (durable job queue)
|
||||
|
||||
outbox (publisher) → читает unpublished из PostgreSQL → публикует в RabbitMQ
|
||||
worker (consumer) → забирает job из RabbitMQ → читает staging → пишет final+variants → status=ready
|
||||
s3_cleanup → сверка S3↔БД, recover зависших processing
|
||||
|
||||
API/worker/outbox → OTLP/HTTP (mTLS) → otel-collector → ┬→ Tempo (traces)
|
||||
├→ Loki (logs)
|
||||
└→ Prometheus (metrics)
|
||||
→ Grafana (дашборд, алерты)
|
||||
|
||||
API/worker → OpenBao (HTTPS, AppRole) → scoped KV v2 secret (TTL 15m)
|
||||
```
|
||||
|
||||
Все сервисы — в одной compose-сети `mtuci2026lab7_default`. Публично опубликованы только LB `127.0.0.1:8443:8443` и Grafana `127.0.0.1:3000:3000`. Реплики `api` **не имеют `ports`** — трафик к ним идёт исключительно через LB по mTLS. PostgreSQL, Redis, MinIO, RabbitMQ, OTel Collector (4317/4318/8888/8889), Prometheus (9090), Loki (3100), Tempo (3200), OpenBao (8200) доступны только по внутренним именам.
|
||||
|
||||
Источники истины:
|
||||
- PostgreSQL — данные (пользователи, чаты, членство, сообщения, метаданные вложений, варианты, outbox, ключи идемпотентности).
|
||||
- Redis — серверные сессии (общие для всех реплик) и счётчики rate-limit входа (общие через реплики).
|
||||
- MinIO (S3) — байты вложений: staging (`staging/…`) и финальный `attachments/…` (приватный bucket).
|
||||
- RabbitMQ — durable очередь job'ов. Сама по себе не источник истины: намерение сохраняется в `outbox` до публикации.
|
||||
- Tempo/Loki/Prometheus — telemetry storage. Их падение не останавливает API.
|
||||
- OpenBao — источник secrets (scoped policy, AppRole, TTL 15m).
|
||||
|
||||
Схема БД (подробнее — `src/app/storage/db.py`, константа `SCHEMA_SQL`):
|
||||
```
|
||||
users(id UUID PK, username TEXT UNIQUE, display_name TEXT, password_hash TEXT, created_at TIMESTAMPTZ)
|
||||
chats(id UUID PK, title TEXT, created_at TIMESTAMPTZ)
|
||||
chat_members(chat_id FK→chats, user_id FK→users, PK(chat_id,user_id))
|
||||
messages(id UUID PK, chat_id FK→chats, sender_id FK→users, text TEXT, created_at TIMESTAMPTZ)
|
||||
+ INDEX idx_messages_chat_created(chat_id, created_at, id)
|
||||
sessions(token TEXT PK, user_id FK→users, expires_at TIMESTAMPTZ) -- fallback SESSION_STORE=postgres
|
||||
idempotency_keys(user_id FK→users, method TEXT, path TEXT, key TEXT,
|
||||
body_hash TEXT, response_status INT, response_body JSONB, created_at TIMESTAMPTZ,
|
||||
PRIMARY KEY (user_id, method, path, key))
|
||||
+ INDEX idx_idempotency_created(created_at)
|
||||
attachments(id UUID PK, chat_id FK→chats, owner_id FK→users,
|
||||
filename TEXT, content_type TEXT, size_bytes BIGINT, sha256 TEXT,
|
||||
object_key TEXT UNIQUE, status TEXT, error_code TEXT,
|
||||
processing_until TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ, updated_at TIMESTAMPTZ)
|
||||
+ INDEX idx_attachments_chat(chat_id)
|
||||
message_attachments(message_id FK→messages, attachment_id FK→attachments, position INT,
|
||||
PRIMARY KEY (message_id, attachment_id))
|
||||
attachment_variants(attachment_id FK→attachments, pipeline_version INT, variant TEXT,
|
||||
object_key TEXT, content_type TEXT, size_bytes BIGINT,
|
||||
width INT, height INT, sha256 TEXT, created_at TIMESTAMPTZ,
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant))
|
||||
outbox(id BIGSERIAL PK, job_id UUID UNIQUE, payload JSONB, created_at TIMESTAMPTZ,
|
||||
published_at TIMESTAMPTZ)
|
||||
+ PARTIAL INDEX idx_outbox_unpublished(id) WHERE published_at IS NULL
|
||||
```
|
||||
|
||||
Применяется идемпотентно через `init_schema()` в отдельном one-shot сервисе `migrate` (`python -m app.storage.db`). Все временные метки — `TIMESTAMPTZ`.
|
||||
|
||||
Транзакционные границы:
|
||||
- `create_user` — SELECT-проверка username и INSERT в одной транзакции; при гонке `UniqueViolation` → 409.
|
||||
- `create_chat` — INSERT `chats` + INSERT `chat_members` в одной транзакции, откат целиком.
|
||||
- `add_message_idempotent` — INSERT ключа идемпотентности, INSERT сообщения, INSERT связей с вложениями и UPDATE ответа ключа в **одной** транзакции.
|
||||
- `create_attachment_queued` — INSERT `attachments` (status=`queued`) + INSERT `outbox` в **одной** транзакции. Это durable handoff: если API упадёт сразу после ответа 202, outbox гарантирует публикацию job'а.
|
||||
- `finalize_ready` — UPDATE `attachments` (status=`ready`, `object_key`=final) + INSERT `attachment_variants` с `ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING` в **одной** транзакции. Идемпотентно к повторной доставке.
|
||||
|
||||
### Дерево проекта (изменения относительно lab6 отмечены `*`)
|
||||
|
||||
```
|
||||
mtuci2026lab7/
|
||||
├── compose.yaml * # + openbao, openbao-init, security_init; TLS/mTLS для всех связей
|
||||
├── Dockerfile * # CMD: uvicorn с --ssl-ca-certs/--ssl-certfile/--ssl-keyfile --ssl-cert-reqs 2
|
||||
├── docker-entrypoint.sh # генерирует INSTANCE_ID, пишет /tmp/instance_id
|
||||
├── certs/ * # CA, серверные и клиентские сертификаты (генерируются security_init)
|
||||
├── secrets/ * # bao-init.json, api-approle.env, worker-approle.env (создаются openbao-init)
|
||||
├── infra/
|
||||
│ ├── nginx.conf * # внешний TLS :8443 + proxy_ssl_* для mTLS к API
|
||||
│ ├── openbao/ * # bao.hcl (file storage, TLS), api-policy.hcl, worker-policy.hcl
|
||||
│ ├── otel/collector.yaml * # TLS/mTLS receivers (client_ca_file)
|
||||
│ ├── prometheus/ # prometheus.yaml, rules/alerts.yaml
|
||||
│ ├── loki/loki.yaml
|
||||
│ ├── tempo/tempo.yaml
|
||||
│ ├── grafana/provisioning/ # datasources, dashboards, alerting
|
||||
│ └── runbooks/ # high-error-rate.md, queue-lag.md, slow-upload.md
|
||||
├── Makefile
|
||||
├── requirements.txt * # + argon2-cffi
|
||||
├── security/THREAT_MODEL.md * # модель угроз (6+ угроз, связь с контролями)
|
||||
├── src/app/
|
||||
│ ├── main.py * # + MAX_REQUEST_BODY_BYTES middleware, Retry-After из HTTPException
|
||||
│ ├── secrets.py * # новый: OpenBao AppRole → scoped token → KV v2 secret
|
||||
│ ├── telemetry.py * # _otel_cert_kwargs: mTLS к Collector (CA + client cert)
|
||||
│ ├── api/auth.py * # rate-limit логина через Redis (общий для реплик)
|
||||
│ ├── api/... # без изменений с lab6
|
||||
│ ├── models/schemas.py
|
||||
│ └── storage/
|
||||
│ ├── passwords.py * # новый: Argon2id (m=19456,t=2,p=1), rehash старого scrypt
|
||||
│ ├── postgres.py * # verify_credentials: Argon2id + rehash
|
||||
│ ├── db.py * # sslmode=verify-full, sslrootcert, statement_timeout
|
||||
│ ├── rabbit.py
|
||||
│ ├── s3.py
|
||||
│ ├── sessions.py
|
||||
│ ├── s3_cleanup.py
|
||||
│ └── replay_failed.py
|
||||
├── scripts/
|
||||
│ ├── restart.sh / .bat * # + OpenBao unseal через openbao-init; HTTPS readiness
|
||||
│ └── stop-one.sh / .bat
|
||||
└── student-tests/
|
||||
└── lab7_test_level4.py * # новый: argon2id, rate-limit, mTLS, OpenBao, hardening, rotation, PG TLS
|
||||
```
|
||||
|
||||
### Что нового в lab7 относительно lab6
|
||||
|
||||
1. **Argon2id** (`src/app/storage/passwords.py`): пароли хранятся как `$argon2id$v=19$m=19456,t=2,p=1$...`. Старые scrypt-хэши принимаются одноразово и при успешном входе перезаписываются Argon2id (rehash).
|
||||
2. **Rate-limit входа**: счётчик `loginrl:<username>:<ip>` в Redis (`INCR`+`EXPIRE`), общий для всех реплик. При превышении — `429` с `Retry-After`.
|
||||
3. **Внешний TLS**: LB слушает `:8443` с `server.crt`/`server.key`, подписанными локальным CA (`security_init`).
|
||||
4. **Внутреннее mTLS LB→API**: API (uvicorn) запущен с `--ssl-cert-reqs 2` (CERT_REQUIRED) и `--ssl-ca-certs /certs/ca.crt`. Nginx предъявляет `lb-client.crt` (SAN `lb-client`), проверяет серверный сертификат API по тому же CA (`proxy_ssl_verify on`, `proxy_ssl_name api`). Прямой публичный API-порт отсутствует.
|
||||
5. **TLS к PostgreSQL**: `sslmode=verify-full` + `sslrootcert=/certs/ca.crt` через `ConnectionPool.kwargs` (DSN остаётся совместимым с прошлыми лабами). PostgreSQL запущен с `ssl=on` и серверным сертификатом.
|
||||
6. **mTLS к OTel Collector**: Collector принимает OTLP по TLS с `client_ca_file=/certs/ca.crt`. API/worker/outbox предъявляют свои клиентские сертификаты (`api-client.crt`, `worker-client.crt`, `outbox-client.crt`).
|
||||
7. **OpenBao вне dev-mode**: file storage, TLS listener на `8200` (внутренний), `security_init` генерирует `bao-server.crt`/`.key`. `openbao-init` (one-shot) делает `operator init`/`unseal`, создаёт KV v2, политики `api-policy`/`worker-policy` и AppRole-роли. RoleID/SecretID пишутся в `/secrets/*-approle.env` (chmod 600). Токены живут 15 минут; root token и unseal key только в `/secrets/bao-init.json`, в API-контейнер не попадают.
|
||||
8. **Scoped policy**: API-токен читает только `secret/data/app/api`, worker — только `secret/data/app/worker`; `sys/policies` и чужой path запрещены.
|
||||
9. **Hardening контейнеров**: API/worker/outbox — non-root, `read_only: true`, `tmpfs: /tmp`, `cap_drop: ALL`, `no-new-privileges:true`. LB — с минимально нужными capabilities (`NET_BIND_SERVICE`, `CHOWN`, `SETGID`, `SETUID`); stateful-сервисы (postgres/minio/rabbitmq) writable — исключение объяснено необходимостью записи данных.
|
||||
10. **Раннее ограничение тела запроса**: middleware в `main.py` по `Content-Length` → `413 payload_too_large` до чтения тела (`MAX_REQUEST_BODY_BYTES`).
|
||||
11. **Ротация leaf-сертификата**: новые leaf сертификаты с тем же SAN `lb-client` генерируются на лету и принимаются API без пересборки образов (проверено `lab7_test_level4.py`).
|
||||
|
||||
### Модель угроз и матрица доверия
|
||||
|
||||
Полная модель — `security/THREAT_MODEL.md`, матрица связей — `contracts/trust-matrix.md` (заполнена). Кратко по активным связям:
|
||||
|
||||
| Связь | Что проверяется | Как |
|
||||
| --- | --- | --- |
|
||||
| Клиент → LB | TLS + hostname (`localhost`/`127.0.0.1`) | `tests/security.py`: с пустым trust store — `SSLCertVerificationError`, не timeout |
|
||||
| LB → API (реплики) | mTLS + identity `lb-client` | `lab7_test_level4.py::test_mtls_lb_to_api`: без cert → reject; чужой self-signed leaf → reject; `lb-client.crt` → OK |
|
||||
| API → PostgreSQL | TLS `verify-full` + CA pin | `db.py` (`sslmode=verify-full`, `sslrootcert`); `pg_stat_ssl` показывает ≥1 SSL-соединение |
|
||||
| API/worker/outbox → Collector | mTLS, разные identities (`api`/`worker`/`outbox`) | `otel-collector` c `client_ca_file`; в приложении `_otel_cert_kwargs` |
|
||||
| API/worker → OpenBao | TLS + AppRole + scoped policy | `secrets.py`; тест: allow `secret/data/app/api`, deny `secret/data/app/worker`, deny `sys/policies` |
|
||||
| API/worker → S3 | TLS + прикладные S3-credentials | MinIO внутренний, `minio-init` создаёт `messenger-app` с ограниченной policy |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities | Внутренний транспорт; в учебном стенде — только compose-сеть (документировано) |
|
||||
| Grafana → datasources | Внутренняя compose-сеть | Datasources не публикуются наружу |
|
||||
| Оператор → OpenBao | Локальный Docker exec/one-shot init | `/secrets/bao-init.json` вне Git (`.dockerignore` исключает `secrets/`) |
|
||||
|
||||
Модель угроз (≥6 связей «угроза → контроль → отрицательный тест»):
|
||||
1. Посторонний пользователь читает metadata/content чужого вложения → ACL в `attachments.py` → `404` в `lab4_test_level4`/`lab7_test_level4`.
|
||||
2. Подделка `X-User-Id` для смены identity → Bearer обязателен, `X-User-Id` игнорируется → smoke (`X-User-Id: <eve>` не меняет владельца).
|
||||
3. Перебор паролей → rate-limit входа (Redis, общий через реплики) → `lab7_test_level4::test_login_rate_limit` (`429`+`Retry-After`).
|
||||
4. Кража/подмена клиента LB → обязательный mTLS + allowlist SAN `lb-client` → `test_mtls_lb_to_api` (без cert / чужой CA → reject).
|
||||
5. Компрометация API-процесса → scoped OpenBao-политика (не читает worker-secret и sys/policies), root token вне контейнера → `test_openbao_policy`.
|
||||
6. Утечка через логи/telemetry → `TraceContextFilter` только технические поля; тела/токены в логи не попадают → `lab6_test_level4::test_logs_in_loki` (нет Authorization/body).
|
||||
7. RCE-подобный доступ в контейнере приложения → `read_only`+`cap_drop ALL`+non-root+`no-new-privileges` → `test_container_hardening`.
|
||||
8. Старый слабый password hash → Argon2id + rehash при входе → `test_argon2id_in_db`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Уровень |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт | `python scripts/check_contract.py` | `PASS: offline skeleton structure…` | 3 |
|
||||
| Полная OpenAPI 3.1 валидация | `python scripts/validate_openapi.py` | `PASS: full OpenAPI 3.1 and JSON Schema validation…` | 3 |
|
||||
| Публичный smoke по HTTPS | `python tests/smoke.py --base-url https://localhost:8443 --ca-file certs/ca.crt` | `PASS: lab7 public API scenario…` | 3 |
|
||||
| Внешний TLS: доверенный CA проходит, недоверенный отвергается | `python tests/security.py --base-url https://localhost:8443 --ca-file ./certs/ca.crt` | `PASS: trusted HTTPS works; untrusted CA rejected…` | 3 |
|
||||
| Сохранность данных и сессии при перезапуске | `python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url https://localhost:8443 --ca-file ./certs/ca.crt` | PASS (hook перезапустил API + хранилища; токен и данные сохранены) | 3/4 |
|
||||
| Отказ одной API-реплики, восстановление и дедупликация | `python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url https://localhost:8443 --ca-file ./certs/ca.crt` | `observed_instances_before=2, successful_writes=8, failed_attempts=0, first_recovery_seconds=6.345`; PASS | 3/4 |
|
||||
| Argon2id, параметры ≥19MiB/t=2/p=1 | `python student-tests/lab7_test_level4.py …` (`test_argon2id_in_db`) | `{'m':'19456','t':'2','p':'1'}` | 3 |
|
||||
| Rate-limit входа `429`+`Retry-After`, общий через Redis | `test_login_rate_limit` | `rate limit hit, Retry-After = 39` | 3 |
|
||||
| mTLS LB→API: positive + 2 negative | `test_mtls_lb_to_api` | `mTLS LB->API: positive + 2 negative passed` | 3 |
|
||||
| Отзыв сессии; пустой trust store → ошибка верификации | `test_session_revocation_and_tls` | `revoke -> 401; empty trust store -> cert verify error` | 3 |
|
||||
| OpenBao scoped policy allow/deny | `test_openbao_policy` | `api can read app/api, cannot read app/worker or sys/policies` | 3 |
|
||||
| Hardening контейнеров API/worker/outbox | `test_container_hardening` | `hardening OK for api/worker/outbox` | 4 |
|
||||
| Ротация leaf-сертификата без пересборки | `test_leaf_cert_rotation` | `fresh leaf accepted by API without rebuild` | 4 |
|
||||
| API→PG TLS `verify-full` + реальные SSL-соединения | `test_pg_tls_verify` | `verify-full + CA pin + actual SSL connections=12` | 3/4 |
|
||||
| Наблюдаемость (traces/metrics/logs/Grafana/alerts) | `student-tests/lab6_test_level4.py` | PASS (перенесено из lab6) | 3/4 |
|
||||
| Фоновые задания (outbox, DLQ, lease, идемпотентность) | `student-tests/lab5_test_level4.py` | PASS (перенесено из lab5) | 3/4 |
|
||||
| Вложения (202/ACL/варианты) | `student-tests/lab4_test_level4.py` | PASS (перенесено из lab4) | 3/4 |
|
||||
| Балансировка/идемпотентность/отказ реплики | `student-tests/lab3_test_level4.py` | PASS (перенесено из lab3) | 3/4 |
|
||||
| Persistence, EXPLAIN, отказ Redis/PG | `student-tests/lab2_test_level4.py` | PASS (перенесено из lab2) | 3/4 |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Выводы тестов
|
||||
|
||||
## Эксперименты и отказы
|
||||
```
|
||||
(venv) PS ...> python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
(venv) PS ...> python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
(venv) PS ...> python tests/smoke.py --base-url https://localhost:8443 --ca-file certs/ca.crt
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
(venv) PS ...> python tests/security.py --base-url https://localhost:8443 --ca-file ./certs/ca.crt
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
(venv) PS ...> python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url https://localhost:8443 --ca-file ./certs/ca.crt
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
(venv) PS ...> python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url https://localhost:8443 --ca-file ./certs/ca.crt
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 8, "failed_attempts": 0, "first_recovery_seconds": 6.345}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
(venv) PS ...> python student-tests/lab7_test_level4.py --base-url https://localhost:8443 --ca-file ./certs/ca.crt
|
||||
=== argon2id in DB ===
|
||||
Argon2id hash params: {'m': '19456', 't': '2', 'p': '1'}
|
||||
PASS: Argon2id hash with m>=19456,t>=2,p>=1
|
||||
=== login rate-limit ===
|
||||
rate limit hit, Retry-After = 39
|
||||
PASS: 429 + Retry-After, shared across replicas via Redis
|
||||
=== mTLS LB -> API ===
|
||||
mTLS LB->API: positive + 2 negative passed
|
||||
PASS: LB identity required, no-cert / foreign-CA rejected
|
||||
=== session revocation over TLS ===
|
||||
PASS: revoke -> 401; empty trust store -> cert verify error
|
||||
=== OpenBao scoped policy ===
|
||||
OpenBao policy: api can read app/api, cannot read app/worker or sys/policies
|
||||
PASS: api token reads only app/api, denied app/worker and sys/policies
|
||||
=== container hardening ===
|
||||
hardening OK for api/worker/outbox
|
||||
PASS: non-root, read_only, cap_drop ALL, no-new-privileges
|
||||
=== leaf cert rotation ===
|
||||
fresh leaf accepted by API without rebuild
|
||||
PASS: fresh lb-client leaf accepted without API rebuild
|
||||
=== API -> PG TLS ===
|
||||
API->PG: verify-full + CA pin + actual SSL connections=12
|
||||
PASS: verify-full + CA pin
|
||||
PASS: all lab7 level-4 checks passed
|
||||
```
|
||||
|
||||
## Ротация, expiry и отзыв credentials (уровень 4)
|
||||
|
||||
**Leaf-сертификат LB** (без пересборки образов):
|
||||
1. `openssl genrsa` + `openssl req -new` + `openssl x509 -req -CA /certs/ca.crt -CAkey /certs/ca.key` для нового CN `lb-client` с тем же SAN.
|
||||
2. Новый сертификат подкладывается контейнеру API во временный файл; API проверяет цепочку по CA и SAN `lb-client` и принимает запрос.
|
||||
3. Старый leaf остаётся валидным до замены на стороне nginx. `test_leaf_cert_rotation` подтверждает приём нового leaf без рестарта/пересборки.
|
||||
|
||||
**Secret API в OpenBao**:
|
||||
- `/secrets/bao-init.json` (root token + unseal key) и `/secrets/*-approle.env` (RoleID/SecretID) не входят в Git (см. `.dockerignore`).
|
||||
- Приложение не хранит root token: `secrets.py` делает AppRole login → short-lived token (TTL 15m) → читает только свой KV path.
|
||||
- Ротация: заменить значение `bao kv put secret/app/api ...`; сервис перечитает при следующем логине (кэш в памяти процесса сбрасывается рестартом/перезапросом токена).
|
||||
- Отзыв identity: `bao token revoke` или `bao write auth/approle/role/api/secret-id-accessor/destroy ...` — новое соединение с отозванным токеном получает `403`; действующие соединения продолжают жить до TTL, что документировано как ожидаемое поведение.
|
||||
|
||||
**Expiry/Sealed OpenBao**:
|
||||
- Если OpenBao запечатан или недоступен, `secrets.py` возвращает `None`; вызывающий код продолжает с уже полученным значением из памяти. При перезапуске и недоступном OpenBao приложение работает с уже прочитанными значениями; `scripts/restart.sh` запускает `openbao-init` для идемпотентного unseal после рестарта.
|
||||
- После `docker compose restart openbao` контейнер возвращается `sealed`; `openbao-init` распечатывает его ключом из `/secrets/bao-init.json`. Unseal-материал не запекается в образ и не выводится в лог.
|
||||
|
||||
**Rate-limit** входа: ключ `loginrl:<username>:<ip>` (IP берётся из `X-Forwarded-For`, который ставит LB). Счётчик обновляется через Redis `INCR`+`EXPIRE`, поэтому лимит общий для реплик; `Retry-After` — остаток TTL.
|
||||
|
||||
## Ограничения и известные упрощения
|
||||
|
||||
- **Учебный стенд на одном Docker-host**: HA к потере хоста не заявлена. Несколько реплик API + LB не делают PostgreSQL/MinIO/RabbitMQ высокодоступными.
|
||||
- **OpenBao bootstrap**: init/unseal автоматизированы в `openbao-init`, но CA private key и unseal share хранятся в `/secrets` на том же хосте. Для реальной эксплуатации нужен внешний bootstrap trust и отдельное хранение unseal (HSM/оператор).
|
||||
- **Redis и MinIO без TLS внутри compose-сети**: документировано как «закрытый локальный transport»; для production-подобной схемы нужен TLS/ACL или sidecar.
|
||||
- **Broker (RabbitMQ)**: внутренний транспорт без mTLS; trust-matrix помечает это как учебное упрощение, ограниченное compose-сетью.
|
||||
- **Ротация секретов**: автоматизирована вручную (повторный `bao kv put` + рестарт или ожидание нового логина); полностью автоматической ротации по таймеру нет.
|
||||
- **Leaf-ротация** проверена на `lb-client`; серверные leaf (`api-server`, `postgres-server`, `collector-server`, `bao-server`) перевыпускаются через `security_init` при отсутствии файлов — без пересборки образов, но с рестартом затронутых сервисов.
|
||||
- **Ранее заявленное (перенесено)**: outbox/DLQ/lease, идемпотентность вложений, отказ PostgreSQL/Redis, наблюдаемость и алерты — работают под HTTPS, что подтверждено smoke и `lab7_test_level4`.
|
||||
+816
@@ -0,0 +1,816 @@
|
||||
x-lab-number: 7
|
||||
x-required-roles:
|
||||
- api
|
||||
- postgres
|
||||
- load_balancer
|
||||
- s3
|
||||
- broker
|
||||
- worker
|
||||
- otel_collector
|
||||
- telemetry_storage
|
||||
- grafana
|
||||
- openbao
|
||||
|
||||
services:
|
||||
# ---------- PKI (one-shot) ----------
|
||||
security_init:
|
||||
image: alpine:3.20
|
||||
container_name: mtuci-security-init
|
||||
user: "0:0"
|
||||
volumes:
|
||||
- ./certs:/certs
|
||||
- ./secrets:/secrets
|
||||
environment:
|
||||
CA_SUBJECT: ${CA_SUBJECT:-/CN=MTUCI-Lab7-CA}
|
||||
SERVER_CN: ${SERVER_CN:-localhost}
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache openssl >/dev/null 2>&1 || true
|
||||
mkdir -p /certs /secrets
|
||||
# --- CA ---
|
||||
if [ ! -f /certs/ca.crt ]; then
|
||||
openssl genrsa -out /certs/ca.key 4096
|
||||
openssl req -x509 -new -nodes -key /certs/ca.key -sha256 -days 3650 \
|
||||
-subj "$${CA_SUBJECT}" -out /certs/ca.crt \
|
||||
-addext "keyUsage=critical,keyCertSign,cRLSign" \
|
||||
-addext "basicConstraints=critical,CA:TRUE"
|
||||
fi
|
||||
# --- LB external server cert ---
|
||||
if [ ! -f /certs/server.crt ]; then
|
||||
openssl genrsa -out /certs/server.key 2048
|
||||
cat > /certs/server.ext <<EOF
|
||||
subjectAltName=DNS:$${SERVER_CN},DNS:localhost,IP:127.0.0.1
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
openssl req -new -key /certs/server.key -subj "/CN=$${SERVER_CN}" -out /certs/server.csr
|
||||
openssl x509 -req -in /certs/server.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/server.crt -days 825 -sha256 -extfile /certs/server.ext
|
||||
fi
|
||||
# --- API server cert ---
|
||||
if [ ! -f /certs/api-server.crt ]; then
|
||||
openssl genrsa -out /certs/api-server.key 2048
|
||||
cat > /certs/api-server.ext <<EOF
|
||||
subjectAltName=DNS:api,DNS:localhost,IP:127.0.0.1
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
openssl req -new -key /certs/api-server.key -subj "/CN=api" -out /certs/api-server.csr
|
||||
openssl x509 -req -in /certs/api-server.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/api-server.crt -days 825 -sha256 -extfile /certs/api-server.ext
|
||||
fi
|
||||
# --- LB client cert ---
|
||||
if [ ! -f /certs/lb-client.crt ]; then
|
||||
openssl genrsa -out /certs/lb-client.key 2048
|
||||
cat > /certs/lb-client.ext <<EOF
|
||||
extendedKeyUsage=clientAuth
|
||||
subjectAltName=DNS:lb-client
|
||||
EOF
|
||||
openssl req -new -key /certs/lb-client.key -subj "/CN=lb-client" -out /certs/lb-client.csr
|
||||
openssl x509 -req -in /certs/lb-client.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/lb-client.crt -days 825 -sha256 -extfile /certs/lb-client.ext
|
||||
fi
|
||||
# --- Client certs: API/worker/outbox -> Collector ---
|
||||
for id in api worker outbox; do
|
||||
if [ ! -f /certs/$${id}-client.crt ]; then
|
||||
openssl genrsa -out /certs/$${id}-client.key 2048
|
||||
cat > /certs/$${id}-client.ext <<EOF
|
||||
extendedKeyUsage=clientAuth
|
||||
subjectAltName=DNS:$${id}
|
||||
EOF
|
||||
openssl req -new -key /certs/$${id}-client.key -subj "/CN=$${id}" -out /certs/$${id}-client.csr
|
||||
openssl x509 -req -in /certs/$${id}-client.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/$${id}-client.crt -days 825 -sha256 -extfile /certs/$${id}-client.ext
|
||||
fi
|
||||
done
|
||||
# --- OpenBao server cert ---
|
||||
if [ ! -f /certs/bao-server.crt ]; then
|
||||
openssl genrsa -out /certs/bao-server.key 2048
|
||||
cat > /certs/bao-server.ext <<EOF
|
||||
subjectAltName=DNS:openbao,DNS:localhost,IP:127.0.0.1
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
openssl req -new -key /certs/bao-server.key -subj "/CN=openbao" -out /certs/bao-server.csr
|
||||
openssl x509 -req -in /certs/bao-server.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/bao-server.crt -days 825 -sha256 -extfile /certs/bao-server.ext
|
||||
fi
|
||||
# --- Collector server cert ---
|
||||
if [ ! -f /certs/collector-server.crt ]; then
|
||||
openssl genrsa -out /certs/collector-server.key 2048
|
||||
cat > /certs/collector-server.ext <<EOF
|
||||
subjectAltName=DNS:otel-collector,DNS:localhost,IP:127.0.0.1
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
openssl req -new -key /certs/collector-server.key -subj "/CN=otel-collector" -out /certs/collector-server.csr
|
||||
openssl x509 -req -in /certs/collector-server.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/collector-server.crt -days 825 -sha256 -extfile /certs/collector-server.ext
|
||||
fi
|
||||
# --- PostgreSQL server cert ---
|
||||
if [ ! -f /certs/postgres-server.crt ]; then
|
||||
openssl genrsa -out /certs/postgres-server.key 2048
|
||||
cat > /certs/postgres-server.ext <<EOF
|
||||
subjectAltName=DNS:postgres,DNS:localhost,IP:127.0.0.1
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
openssl req -new -key /certs/postgres-server.key -subj "/CN=postgres" -out /certs/postgres-server.csr
|
||||
openssl x509 -req -in /certs/postgres-server.csr -CA /certs/ca.crt -CAkey /certs/ca.key \
|
||||
-CAcreateserial -out /certs/postgres-server.crt -days 825 -sha256 -extfile /certs/postgres-server.ext
|
||||
fi
|
||||
# Права: сервисы в контейнерах работают от разных UID (postgres, openbao и т.п.),
|
||||
# поэтому cert/key должны быть читаемы как обычные файлы. Учебный стенд → 644.
|
||||
# ИСКЛЮЧЕНИЕ: PostgreSQL требует 0600 И владельца postgres. chown ниже работает,
|
||||
# но bind-mount из Windows (NTFS) может терять UID — поэтому postgres
|
||||
# использует entrypoint-wrapper, который копирует ключ в свой container fs.
|
||||
chmod 644 /certs/*.crt 2>/dev/null || true
|
||||
chmod 644 /certs/*.key 2>/dev/null || true
|
||||
chmod 600 /certs/ca.key 2>/dev/null || true
|
||||
chown 999:999 /certs/postgres-server.key 2>/dev/null || true
|
||||
chmod 600 /certs/postgres-server.key 2>/dev/null || true
|
||||
echo "event=pki_ready"
|
||||
|
||||
# ---------- OpenBao (non-dev, file storage, TLS) ----------
|
||||
openbao:
|
||||
image: openbao/openbao:2.0.2
|
||||
container_name: mtuci-openbao
|
||||
# lab7 fix: НЕ используем init: true — tini как PID 1 не освобождает
|
||||
# порт при рестартах, старый bao остаётся жив и получается
|
||||
# "bind: address already in use". Без tini bao - PID 1 и корректно
|
||||
# завершается по SIGTERM.
|
||||
#
|
||||
# lab7 fix: явный entrypoint — образ openbao/openbao:2.0.2 уже имеет
|
||||
# ENTRYPOINT ["bao"], но при command: ["server", "-config=..."] в
|
||||
# некоторых версиях Docker Desktop получалось дублирование команды
|
||||
# (`bao server bao server ...`), и второй bao падал с
|
||||
# "address already in use". Явный entrypoint исключает это.
|
||||
entrypoint: ["bao"]
|
||||
depends_on:
|
||||
security_init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
SKIP_SETCAP: "1"
|
||||
volumes:
|
||||
- ./infra/openbao/bao.hcl:/openbao/config/bao.hcl:ro
|
||||
- ./certs:/certs:ro
|
||||
- openbao-data:/openbao/data
|
||||
command: ["server", "-config=/openbao/config/bao.hcl"]
|
||||
# lab7: порт 8200 НЕ публикуем наружу — сервис доступен только по
|
||||
# внутреннему DNS-имени openbao в compose-сети.
|
||||
#
|
||||
# lab7 fix: healthcheck через TCP-проверку listener'а. `bao status`
|
||||
# при sealed возвращает exit 2 → deadlock: openbao-init ждёт healthy,
|
||||
# а healthy требует unsealed. `nc -z` проверяет только "порт открыт".
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "nc -z 127.0.0.1 8200"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
restart: on-failure:3
|
||||
stop_grace_period: 15s
|
||||
stop_signal: SIGTERM
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 64M
|
||||
|
||||
openbao-init:
|
||||
image: openbao/openbao:2.0.2
|
||||
container_name: mtuci-openbao-init
|
||||
# lab7 fix: jq нужен для разбора /secrets/bao-init.json, но в образе
|
||||
# openbao/openbao:2.0.2 его нет. Ставим через apk — для этого нужен
|
||||
# root. Контейнер одноразовый bootstrap, root здесь уместен.
|
||||
user: "0:0"
|
||||
depends_on:
|
||||
openbao:
|
||||
condition: service_healthy
|
||||
security_init:
|
||||
condition: service_completed_successfully
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
- ./secrets:/secrets
|
||||
- ./infra/openbao:/openbao-config:ro
|
||||
environment:
|
||||
BAO_ADDR: ${BAO_ADDR:-https://openbao:8200}
|
||||
BAO_CACERT: /certs/ca.crt
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache jq >/dev/null 2>&1 || true
|
||||
export BAO_CACERT=/certs/ca.crt
|
||||
|
||||
if [ -f /secrets/bao-init.json ]; then
|
||||
echo "event=bao_already_initialized"
|
||||
else
|
||||
bao operator init -key-shares=1 -key-threshold=1 \
|
||||
-format=json > /secrets/bao-init.json
|
||||
chmod 600 /secrets/bao-init.json
|
||||
echo "event=bao_initialized"
|
||||
fi
|
||||
|
||||
UNSEAL=$$(jq -r '.unseal_keys_b64[0]' /secrets/bao-init.json)
|
||||
ROOT=$$(jq -r '.root_token' /secrets/bao-init.json)
|
||||
bao operator unseal "$$UNSEAL" >/dev/null 2>&1 || true
|
||||
export BAO_TOKEN="$$ROOT"
|
||||
|
||||
bao secrets enable -path=secret kv-v2 2>/dev/null || true
|
||||
bao policy write api-policy /openbao-config/api-policy.hcl
|
||||
bao policy write worker-policy /openbao-config/worker-policy.hcl
|
||||
bao auth enable approle 2>/dev/null || true
|
||||
bao write auth/approle/role/api token_policies=api-policy \
|
||||
token_ttl=15m token_max_ttl=1h secret_id_ttl=15m secret_id_num_uses=10 2>/dev/null || true
|
||||
bao write auth/approle/role/worker token_policies=worker-policy \
|
||||
token_ttl=15m token_max_ttl=1h secret_id_ttl=15m secret_id_num_uses=10 2>/dev/null || true
|
||||
|
||||
# lab7 fix: политики разрешают чтение secret/data/app/{api,worker},
|
||||
# но самих секретов до этого никто не создавал -> read возвращал 404
|
||||
# и тест считал, что allow сломан. Записываем демонстрационные
|
||||
# статические KV v2 секреты (учебные, не секретные значения).
|
||||
# Идемпотентно: повторный put перезаписывает то же значение.
|
||||
bao kv put secret/app/api \
|
||||
db_password="lab-api-db-not-a-secret" \
|
||||
s3_secret="lab-api-s3-not-a-secret" \
|
||||
session_pepper="lab-api-pepper-not-a-secret" >/dev/null
|
||||
bao kv put secret/app/worker \
|
||||
db_password="lab-worker-db-not-a-secret" \
|
||||
s3_secret="lab-worker-s3-not-a-secret" >/dev/null
|
||||
|
||||
API_ROLE_ID=$$(bao read -field=role_id auth/approle/role/api/role-id)
|
||||
API_SECRET_ID=$$(bao write -f -field=secret_id auth/approle/role/api/secret-id)
|
||||
WORKER_ROLE_ID=$$(bao read -field=role_id auth/approle/role/worker/role-id)
|
||||
WORKER_SECRET_ID=$$(bao write -f -field=secret_id auth/approle/role/worker/secret-id)
|
||||
printf "API_ROLE_ID=$$API_ROLE_ID\nAPI_SECRET_ID=$$API_SECRET_ID\n" > /secrets/api-approle.env
|
||||
printf "WORKER_ROLE_ID=$$WORKER_ROLE_ID\nWORKER_SECRET_ID=$$WORKER_SECRET_ID\n" > /secrets/worker-approle.env
|
||||
chmod 600 /secrets/api-approle.env /secrets/worker-approle.env
|
||||
echo "event=openbao_bootstrap_done"
|
||||
restart: "no"
|
||||
|
||||
# ---------- Хранилища ----------
|
||||
postgres:
|
||||
image: postgres:18.6-alpine
|
||||
container_name: mtuci-postgres
|
||||
depends_on:
|
||||
security_init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-high_load_messenger}
|
||||
# lab7: bind-mount из Windows (NTFS/WSL2) не сохраняет UID владельца,
|
||||
# поэтому postmaster видит postgres-server.key как "чужой". Решение:
|
||||
# entrypoint копирует certs из read-only bind в /var/lib/postgresql/certs
|
||||
# (обычная container-fs) и выставляет 0600/0644 + chown postgres:postgres.
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mkdir -p /var/lib/postgresql/certs
|
||||
cp /etc/postgres-certs-src/*.crt /etc/postgres-certs-src/*.key /var/lib/postgresql/certs/
|
||||
chown -R postgres:postgres /var/lib/postgresql/certs
|
||||
chmod 644 /var/lib/postgresql/certs/*.crt
|
||||
chmod 600 /var/lib/postgresql/certs/*.key
|
||||
exec docker-entrypoint.sh postgres \
|
||||
-c ssl=on \
|
||||
-c ssl_cert_file=/var/lib/postgresql/certs/postgres-server.crt \
|
||||
-c ssl_key_file=/var/lib/postgresql/certs/postgres-server.key \
|
||||
-c ssl_ca_file=/var/lib/postgresql/certs/ca.crt
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql
|
||||
- ./certs:/etc/postgres-certs-src:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-high_load_messenger}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 30s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
container_name: mtuci-redis
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"]
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 3s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 64M
|
||||
|
||||
minio:
|
||||
image: netvark/minio:20260213
|
||||
container_name: mtuci-minio
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
|
||||
MINIO_BROWSER_REDIRECT_URL: http://localhost:9001
|
||||
volumes:
|
||||
- minio-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 128M
|
||||
|
||||
minio-init:
|
||||
image: netvark/mc:20260920
|
||||
container_name: mtuci-minio-init
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}"
|
||||
mc mb --ignore-existing local/"$${S3_BUCKET}"
|
||||
mc anonymous set none local/"$${S3_BUCKET}" || true
|
||||
if ! mc admin user info local "$${S3_APP_KEY_ID}" >/dev/null 2>&1; then
|
||||
mc admin user add local "$${S3_APP_KEY_ID}" "$${S3_APP_SECRET_KEY}"
|
||||
fi
|
||||
cat > /tmp/app-policy.json <<'POLICY'
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::${S3_BUCKET}"]},
|
||||
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], "Resource": ["arn:aws:s3:::${S3_BUCKET}/*"]}
|
||||
]
|
||||
}
|
||||
POLICY
|
||||
sed -i "s/\$${S3_BUCKET}/$${S3_BUCKET}/g" /tmp/app-policy.json || true
|
||||
mc admin policy create local messenger-app /tmp/app-policy.json 2>/dev/null || true
|
||||
mc admin policy attach local messenger-app --user "$${S3_APP_KEY_ID}" || true
|
||||
echo "minio-init done"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_APP_KEY_ID: ${S3_APP_KEY_ID:-messenger-app}
|
||||
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY:-messenger-app-secret-change-me}
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.2"
|
||||
memory: 64M
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtuci-messenger:lab7
|
||||
command: ["python", "-m", "app.storage.db"]
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
SESSION_STORE: ${SESSION_STORE:-redis}
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
TLS_CA_FILE: /certs/ca.crt
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 64M
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.13.7-management-alpine
|
||||
container_name: mtuci-rabbitmq
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-messenger-rabbit-change-me}
|
||||
volumes:
|
||||
- rabbitmq-data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 128M
|
||||
|
||||
worker:
|
||||
build: .
|
||||
image: mtuci-messenger:lab7
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_APP_KEY_ID: ${S3_APP_KEY_ID}
|
||||
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY}
|
||||
RABBITMQ_URL: ${RABBITMQ_URL:-amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F}
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /certs/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /certs/worker-client.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /certs/worker-client.key
|
||||
BAO_ADDR: ${BAO_ADDR:-https://openbao:8200}
|
||||
BAO_CACERT: ${BAO_CACERT:-/certs/ca.crt}
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
- ./secrets:/secrets:ro
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
openbao-init:
|
||||
condition: service_completed_successfully
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
outbox:
|
||||
build: .
|
||||
image: mtuci-messenger:lab7
|
||||
command: ["python", "-m", "app.outbox_worker"]
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
RABBITMQ_URL: ${RABBITMQ_URL:-amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F}
|
||||
OTEL_SERVICE_NAME: messenger-outbox
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /certs/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /certs/outbox-client.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /certs/outbox-client.key
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.3"
|
||||
memory: 128M
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtuci-messenger:lab7
|
||||
environment:
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
SESSION_STORE: ${SESSION_STORE:-redis}
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_APP_KEY_ID: ${S3_APP_KEY_ID}
|
||||
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_REQUEST_BODY_BYTES: ${MAX_REQUEST_BODY_BYTES:-10485760}
|
||||
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300}
|
||||
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-3600}
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /certs/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /certs/api-client.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /certs/api-client.key
|
||||
ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456}
|
||||
ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2}
|
||||
ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1}
|
||||
LOGIN_WINDOW_SECONDS: ${LOGIN_WINDOW_SECONDS:-60}
|
||||
LOGIN_MAX_ATTEMPTS: ${LOGIN_MAX_ATTEMPTS:-10}
|
||||
BAO_ADDR: ${BAO_ADDR:-https://openbao:8200}
|
||||
BAO_CACERT: ${BAO_CACERT:-/certs/ca.crt}
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
- ./secrets:/secrets:ro
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
openbao-init:
|
||||
condition: service_completed_successfully
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
# lab7 fix: uvicorn запущен с --ssl-cert-reqs 2 (CERT_REQUIRED, mTLS).
|
||||
# Без клиентского сертификата сервер рвёт соединение до ответа
|
||||
# (RemoteDisconnected). Используем ровно ту identity, которой
|
||||
# ходит LB: /certs/lb-client.crt + lb-client.key, CA=/certs/ca.crt.
|
||||
test: ["CMD", "python", "-c", "import ssl,urllib.request; ctx=ssl.create_default_context(cafile='/certs/ca.crt'); ctx.load_cert_chain(certfile='/certs/lb-client.crt', keyfile='/certs/lb-client.key'); urllib.request.urlopen('https://127.0.0.1:8080/health/ready', context=ctx, timeout=3).read()"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
replicas: 2
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
s3_cleanup:
|
||||
build: .
|
||||
image: mtuci-messenger:lab7
|
||||
command: ["python", "-m", "app.storage.s3_cleanup"]
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_APP_KEY_ID: ${S3_APP_KEY_ID}
|
||||
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY}
|
||||
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300}
|
||||
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-3600}
|
||||
TLS_CA_FILE: /certs/ca.crt
|
||||
volumes:
|
||||
- ./certs:/certs:ro
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.3"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.05"
|
||||
memory: 64M
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.3-alpine
|
||||
container_name: mtuci-lb
|
||||
ports:
|
||||
- "127.0.0.1:8443:8443"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
- ./certs:/etc/nginx/certs:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /var/cache/nginx
|
||||
- /var/run
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- NET_BIND_SERVICE
|
||||
- CHOWN
|
||||
# lab7 fix: nginx master стартует от root и сбрасывает привилегии
|
||||
# воркеров до пользователя nginx (UID 101). Для setgid/setuid нужны
|
||||
# соответствующие capabilities, иначе воркеры падают с
|
||||
# "setgid(101) failed (1: Operation not permitted)".
|
||||
- SETGID
|
||||
- SETUID
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 32M
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.108.0
|
||||
container_name: mtuci-otel-collector
|
||||
command: ["--config=/etc/otel/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro
|
||||
- ./certs:/certs:ro
|
||||
depends_on:
|
||||
- tempo
|
||||
- loki
|
||||
- prometheus
|
||||
healthcheck:
|
||||
test: ["CMD", "/otelcol-contrib", "--version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 64M
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.54.1
|
||||
container_name: mtuci-prometheus
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yaml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time=15d"
|
||||
- "--web.listen-address=0.0.0.0:9090"
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- prometheus-data:/prometheus
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 128M
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.1.1
|
||||
container_name: mtuci-loki
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki-data:/loki
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 128M
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.5.0
|
||||
container_name: mtuci-tempo
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo-data:/var/tempo
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 128M
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.2.0
|
||||
container_name: mtuci-grafana
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor"
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.3"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.05"
|
||||
memory: 64M
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
name: mtuci2026lab7_postgres-data
|
||||
redis-data:
|
||||
name: mtuci2026lab7_redis-data
|
||||
minio-data:
|
||||
name: mtuci2026lab7_minio-data
|
||||
rabbitmq-data:
|
||||
name: mtuci2026lab7_rabbitmq-data
|
||||
prometheus-data:
|
||||
name: mtuci2026lab7_prometheus-data
|
||||
loki-data:
|
||||
name: mtuci2026lab7_loki-data
|
||||
tempo-data:
|
||||
name: mtuci2026lab7_tempo-data
|
||||
grafana-data:
|
||||
name: mtuci2026lab7_grafana-data
|
||||
openbao-data:
|
||||
name: mtuci2026lab7_openbao-data
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: mtuci2026lab7_default
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# X-Instance-Id: если INSTANCE_ID не задан снаружи, генерируем уникальный
|
||||
# для этого контейнера при старте. Кладём его в файл /tmp/instance_id,
|
||||
# чтобы stop-one.bat/.sh могли сопоставить alias и контейнер.
|
||||
if [ -z "${INSTANCE_ID:-}" ]; then
|
||||
INSTANCE_ID=$(python -c "import uuid; print(uuid.uuid4().hex[:12])")
|
||||
export INSTANCE_ID
|
||||
fi
|
||||
echo "event=instance_id instance_id=${INSTANCE_ID}"
|
||||
echo -n "${INSTANCE_ID}" > /tmp/instance_id
|
||||
|
||||
exec "$@"
|
||||
@@ -0,0 +1,58 @@
|
||||
|
||||
=== Лабораторная 1. Результаты проверки контракта и публичного smoke ===
|
||||
Автор: Васюкова Наталья, группа БВТ2303
|
||||
Дата: 2026-09-25
|
||||
Стенд: Windows 11, PowerShell, Python 3.14.5 (venv), Docker Desktop
|
||||
Образ: mtusi-messenger:lab1 (python:3.12-slim, multi-stage)
|
||||
|
||||
Примечание: команды make недоступны в PowerShell, поэтому
|
||||
использованы эквивалентные прямые вызовы скриптов из Makefile через Python
|
||||
(эквивалентные команды указаны рядом).
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
1. make check -> python scripts/check_contract.py
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
Наблюдение: структура репозитория целостна (скелет не изменён): lab.json и версия в openapi.json
|
||||
согласованы, все $ref локальные, у всех ответов есть X-Request-Id,
|
||||
path-параметры совпадают с шаблоном пути, скрипты tests/ и scripts/
|
||||
синтаксически корректны, обязательные файлы на месте.
|
||||
|
||||
--------------------------------------------------------------------
|
||||
2. make validate -> python scripts/validate_openapi.py
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
|
||||
Наблюдение: контракт contracts/openapi.json прошёл полную валидацию по OpenAPI 3.1,
|
||||
примеры из contracts/examples.json соответствуют схемам компонентов.
|
||||
|
||||
--------------------------------------------------------------------
|
||||
3. make test -> python tests/smoke.py --base-url http://localhost:8080
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ python tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab1 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
Наблюдение: сквозной публичный сценарий ЛР1 пройден. Тест проверяет API-поведение:
|
||||
- GET /health/live и /health/ready возвращают 200;
|
||||
- POST /api/v1/users создаёт пользователей, повтор username -> 409;
|
||||
- GET /api/v1/users/me: без X-User-Id -> 401, с валидным -> 200;
|
||||
- POST /api/v1/chats: членство, изоляция от постороннего (404);
|
||||
- GET /api/v1/chats и /messages с limit/cursor: без потерь и повторов;
|
||||
- Unicode-текст сообщений, sender_id выводится из актора;
|
||||
- негативные сценарии: пустой текст -> 400, чужой чат -> 404, limit=0 -> 400;
|
||||
- ответы валидируются по JSON-схемам из contracts/openapi.json.
|
||||
Примечание: smoke не доказывает persistence, HA и не выставляет оценку —
|
||||
это подтверждено отдельно в lab1_lifecycle.txt.
|
||||
|
||||
--------------------------------------------------------------------
|
||||
ИТОГ
|
||||
--------------------------------------------------------------------
|
||||
|
||||
Все три проверки пройдены.
|
||||
@@ -0,0 +1,155 @@
|
||||
=== Лабораторная 1. Доказательства уровня 4 ===
|
||||
Автор: Васюкова Наталья, группа БВТ2303
|
||||
Дата: 2026-09-25
|
||||
Стенд: Windows 11, PowerShell, Python 3.12 (venv), Docker Desktop
|
||||
Образ: mtusi-messenger:lab1 (python:3.12-slim, multi-stage)
|
||||
Commit: solution/BVT2303-Vasyukova
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
1. Собственные негативные тесты
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ python student-tests/test_negative.py --base-url http://localhost:8080
|
||||
PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, wrong type,
|
||||
empty text, foreign chat, pagination, bad limit, unknown user)
|
||||
|
||||
Наблюдение: проверены сценарии — повтор username -> 409; malformed JSON -> 400;
|
||||
лишние поля -> 400; неверный тип поля -> 400; пустое/пробельное сообщение -> 400;
|
||||
доступ к чужому чату (GET чата, GET сообщений, POST сообщения) -> 404;
|
||||
обход нескольких страниц cursor-пагинации без потерь, дублей и изменения порядка;
|
||||
limit=0 и limit=101 -> 400; неизвестный UUID в X-User-Id -> 401.
|
||||
Все ответы об ошибках соответствуют формату contracts/openapi.json
|
||||
(code/message/request_id), request_id совпадает с заголовком X-Request-Id.
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
2. Конкурентный тест: 100 сообщений от 10 клиентов
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ python student-tests/test_concurrency.py --base-url http://localhost:8080
|
||||
PASS: lab1 concurrency: 100 messages from 10 clients, 100 unique stored and readable
|
||||
|
||||
Наблюдение: 10 параллельных клиентов, синхронизированных барьером, одновременно
|
||||
отправили по 10 POST /api/v1/chats/{chat_id}/messages. Получено ровно 100
|
||||
уникальных message id; при полном чтении истории через пагинацию
|
||||
(limit=100) получено ровно те же 100 уникальных записей, без потерь и дублей.
|
||||
Это подтверждает атомарность операций MemoryStorage под threading.RLock.
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
3. Единая команда запуска собственных тестов
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ python student-tests/run_all.py --base-url http://localhost:8080
|
||||
=== test_negative.py ===
|
||||
PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, wrong type, empty text, foreign chat, pagination, bad limit, unknown user)
|
||||
=== test_concurrency.py ===
|
||||
PASS: lab1 concurrency: 100 messages from 10 clients, 100 unique stored and readable
|
||||
PASS: all student tests (level 4) passed
|
||||
|
||||
(Эквивалент через Makefile: make test-student BASE_URL=http://localhost:8080)
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
4. Обработка SIGTERM
|
||||
--------------------------------------------------------------------
|
||||
|
||||
В терминале 1:
|
||||
$ docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
|
||||
В терминале 2:
|
||||
Цикл опроса TCP-порта 8080 (TcpClient, жёсткий таймаут 500 мс):
|
||||
```
|
||||
while ($true) {
|
||||
$t = Get-Date -Format "HH:mm:ss.fff"
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try {
|
||||
$iar = $client.BeginConnect("127.0.0.1", 8080, $null, $null)
|
||||
$ok = $iar.AsyncWaitHandle.WaitOne(500)
|
||||
if ($ok -and $client.Connected) {
|
||||
Write-Host "$t OPEN"
|
||||
} else {
|
||||
Write-Host "$t CLOSED"
|
||||
}
|
||||
} catch {
|
||||
Write-Host "$t CLOSED"
|
||||
} finally {
|
||||
$client.Close()
|
||||
}
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
```
|
||||
В терминале 3:
|
||||
$ docker stop -t 10 mtusi-messenger-lab1
|
||||
Замер времени завершения:
|
||||
$ $t = Measure-Command { docker stop -t 10 mtusi-messenger-lab1 }
|
||||
$ "Stop took: $($t.TotalSeconds) s"
|
||||
Stop took: 0.7416883 s
|
||||
|
||||
Фрагмент вывода терминала 2:
|
||||
02:06:06.269 OPEN
|
||||
02:06:06.497 CLOSED
|
||||
|
||||
Фрагмент лога контейнера:
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
2026-09-25 23:06:05,866 INFO app event=shutdown_complete
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [1]
|
||||
|
||||
Наблюдение: контейнер завершился за ~0.74 с (существенно быстрее grace-period 10 с).
|
||||
Порт закрылся (первый CLOSED) через ~0.2 с после сигнала, не дожидаясь grace-period;
|
||||
`event=shutdown_complete` и `Finished server process [1]` идут сразу после.
|
||||
Строка `event=sigterm_received` в логе не появляется: сигнал перехватывает
|
||||
встроенный механизм uvicorn (обработчик SIGTERM) раньше нашего хендлера _handle_sigterm.
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
5. Логи без содержимого сообщений
|
||||
--------------------------------------------------------------------
|
||||
|
||||
Пример строк лога контейнера (meta-only):
|
||||
|
||||
2026-09-25 ... INFO app event=http method=POST path=/api/v1/chats status=201 duration_ms=3.12 request_id=...
|
||||
2026-09-25 ... INFO app event=http method=POST path=/api/v1/chats/<uuid>/messages status=201 duration_ms=2.01 request_id=...
|
||||
2026-09-25 ... INFO app event=startup app_env=lab
|
||||
2026-09-25 ... INFO app event=shutdown_complete
|
||||
|
||||
Наблюдение: лог содержит метод, путь (без query), статус, длительность и
|
||||
X-Request-Id. Текст сообщений, значения X-User-Id, пароли и токены в лог
|
||||
не попадают. Ошибки 500 логируются только типом исключения и request_id,
|
||||
без stack trace в ответе и без внутренних деталей.
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
6. Конфигурация контейнера (уровень 4)
|
||||
--------------------------------------------------------------------
|
||||
|
||||
- Dockerfile: базовый образ python:3.12-slim (multi-stage), зависимости
|
||||
закреплены (fastapi==0.115.0, uvicorn[standard]==0.32.0).
|
||||
- Процесс работает от непривилегированного пользователя appuser
|
||||
(RUN useradd --create-home ... ; USER appuser).
|
||||
- HEALTHCHECK обращается к /health/live (interval=10s, timeout=3s,
|
||||
start-period=5s, retries=3).
|
||||
- .dockerignore исключает .env, secrets, certs, volumes, backups,
|
||||
*.key/*.pem/*.p12/*.pfx/*.jks, .venv, __pycache__, node_modules,
|
||||
evidence, .DS_Store. В образ не копируются локальные секреты.
|
||||
|
||||
Проверка:
|
||||
$ docker inspect --format "{{.Config.User}}" mtusi-messenger-lab1
|
||||
appuser
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
ИТОГ
|
||||
--------------------------------------------------------------------
|
||||
|
||||
Результаты:
|
||||
- собственные негативные тесты — PASS;
|
||||
- конкурентный тест 100/10 — PASS, 100 уникальных сообщений;
|
||||
- единая команда запуска test-student — PASS;
|
||||
- SIGTERM: контейнер останавливается за ~0.74 с, порт закрывается через
|
||||
~0.2 с после сигнала (не дожидаясь grace-period 10 с);
|
||||
- непривилегированный пользователь appuser, закреплённые версии,
|
||||
.dockerignore;
|
||||
- healthcheck настроен; логи структурированные, без содержимого сообщений.
|
||||
@@ -0,0 +1,58 @@
|
||||
=== Лабораторная 1. Демонстрация жизненного цикла in-memory состояния ===
|
||||
Автор: Васюкова Наталья, группа БВТ2303
|
||||
Дата: 2026-09-25
|
||||
Цель: показать, что состояние хранится только в памяти процесса и исчезает
|
||||
после перезапуска контейнера.
|
||||
|
||||
|
||||
--------------------------------------------------------------------
|
||||
1. До перезапуска: пользователь существует
|
||||
--------------------------------------------------------------------
|
||||
|
||||
Пользователь создан через Swagger. Получен его id.
|
||||
|
||||
Проверка:
|
||||
$ curl.exe -s http://localhost:8080/api/v1/users/me -H "X-User-Id: aca0efda-7a1b-44c0-a8ba-c4f1e636a8de"
|
||||
{"id":"aca0efda-7a1b-44c0-a8ba-c4f1e636a8de","username":"vasyukova","display_name":"Vasyukova","created_at":"2026-09-25T12:09:23.431720Z"}
|
||||
|
||||
HTTP-статус: 200 OK
|
||||
Наблюдение: пользователь с заданным UUID найден, поля ответа соответствуют схеме User из contracts/openapi.json.
|
||||
|
||||
--------------------------------------------------------------------
|
||||
2. Остановка контейнера
|
||||
--------------------------------------------------------------------
|
||||
|
||||
В терминале с запущенным контейнером:
|
||||
Ctrl+C.
|
||||
Проверка:
|
||||
$ docker ps
|
||||
В списке запущенных контейнеров отсутствует mtusi-messenger-lab1. Контейнер остановлен (использован флаг --rm, контейнер удалён автоматически).
|
||||
|
||||
--------------------------------------------------------------------
|
||||
3. Повторный запуск контейнера
|
||||
--------------------------------------------------------------------
|
||||
|
||||
$ docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
|
||||
Uvicorn запущен на http://0.0.0.0:8080, контейнер готов принимать запросы.
|
||||
|
||||
--------------------------------------------------------------------
|
||||
4. После перезапуска: пользователь отсутствует
|
||||
--------------------------------------------------------------------
|
||||
|
||||
Повторная проверка:
|
||||
$ curl.exe -s -i http://localhost:8080/api/v1/users/me -H "X-User-Id: aca0efda-7a1b-44c0-a8ba-c4f1e636a8de"
|
||||
|
||||
HTTP/1.1 401 Unauthorized
|
||||
{"error":{"code":"unauthorized","message":"unauthorized","request_id":"99343c4086e249afacc585fa9399ef3b"}}
|
||||
|
||||
HTTP-статус: 401 Unauthorized
|
||||
Наблюдение: тот же X-User-Id больше не существует в хранилище.
|
||||
Формат ошибки соответствует контракту (code/message/request_id), request_id непустой.
|
||||
|
||||
--------------------------------------------------------------------
|
||||
ВЫВОД
|
||||
--------------------------------------------------------------------
|
||||
|
||||
После перезапуска пользователь исчез. Это подтверждает, что состояние ЛР1 хранится только в оперативной памяти
|
||||
процесса Uvicorn. Volumes, БД, файлы состояния не используются. Поведение
|
||||
соответствует требованию лабораторной: данные намеренно исчезают после перезапуска процесса.
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"lab": 1,
|
||||
"track": 1,
|
||||
"variant": "after",
|
||||
"profile": {
|
||||
"history_sizes": [100, 1000, 5000],
|
||||
"concurrency_levels": [1, 4, 16],
|
||||
"duration_seconds": 10,
|
||||
"note": "closed-loop; клиент ограничивает throughput; RSS — docker stats; sort on read REMOVED"
|
||||
},
|
||||
"points": [
|
||||
{"history_size": 100, "concurrency": 1, "duration_s": 10.357, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2064.323, "p95": 2078.936, "p99": 2078.936}, "rss_mb": 46.01},
|
||||
{"history_size": 100, "concurrency": 4, "duration_s": 10.379, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.927, "latency_ms": {"p50": 2061.138, "p95": 2090.765, "p99": 2093.1}, "rss_mb": 35.47},
|
||||
{"history_size": 100, "concurrency": 16, "duration_s": 10.522, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.603, "latency_ms": {"p50": 2065.596, "p95": 2084.327, "p99": 2090.339}, "rss_mb": 35.54},
|
||||
|
||||
{"history_size": 1000, "concurrency": 1, "duration_s": 10.341, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2070.498, "p95": 2076.896, "p99": 2076.896}, "rss_mb": 35.91},
|
||||
{"history_size": 1000, "concurrency": 4, "duration_s": 10.383, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.926, "latency_ms": {"p50": 2061.585, "p95": 2085.476, "p99": 2089.531}, "rss_mb": 36.91},
|
||||
{"history_size": 1000, "concurrency": 16, "duration_s": 10.571, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.568, "latency_ms": {"p50": 2075.703, "p95": 2092.981, "p99": 2102.494}, "rss_mb": 37.53},
|
||||
|
||||
{"history_size": 5000, "concurrency": 1, "duration_s": 10.296, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.486, "latency_ms": {"p50": 2043.537, "p95": 2086.686, "p99": 2086.686}, "rss_mb": 41.6},
|
||||
{"history_size": 5000, "concurrency": 4, "duration_s": 10.324, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.937, "latency_ms": {"p50": 2057.056, "p95": 2065.708, "p99": 2068.095}, "rss_mb": 42.15},
|
||||
{"history_size": 5000, "concurrency": 16, "duration_s": 10.747, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.444, "latency_ms": {"p50": 2072.015, "p95": 2116.62, "p99": 2124.503}, "rss_mb": 43.08}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"lab": 1,
|
||||
"track": 1,
|
||||
"variant": "before",
|
||||
"profile": {
|
||||
"history_sizes": [100, 1000, 5000],
|
||||
"concurrency_levels": [1, 4, 16],
|
||||
"duration_seconds": 10,
|
||||
"note": "closed-loop; клиент ограничивает throughput; RSS — docker stats; sort on read ENABLED"
|
||||
},
|
||||
"points": [
|
||||
{"history_size": 100, "concurrency": 1, "duration_s": 10.355, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2066.139, "p95": 2085.291, "p99": 2085.291}, "rss_mb": 34.35},
|
||||
{"history_size": 100, "concurrency": 4, "duration_s": 10.415, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.92, "latency_ms": {"p50": 2073.966, "p95": 2088.797, "p99": 2091.95}, "rss_mb": 34.83},
|
||||
{"history_size": 100, "concurrency": 16, "duration_s": 10.589, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.555, "latency_ms": {"p50": 2071.131, "p95": 2127.922, "p99": 2134.262}, "rss_mb": 35.44},
|
||||
|
||||
{"history_size": 1000, "concurrency": 1, "duration_s": 10.372, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.482, "latency_ms": {"p50": 2076.358, "p95": 2082.009, "p99": 2082.009}, "rss_mb": 36.84},
|
||||
{"history_size": 1000, "concurrency": 4, "duration_s": 10.381, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.927, "latency_ms": {"p50": 2062.515, "p95": 2094.389, "p99": 2106.044}, "rss_mb": 36.71},
|
||||
{"history_size": 1000, "concurrency": 16, "duration_s": 10.58, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.562, "latency_ms": {"p50": 2076.528, "p95": 2101.717, "p99": 2115.533}, "rss_mb": 38.19},
|
||||
|
||||
{"history_size": 5000, "concurrency": 1, "duration_s": 10.39, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.481, "latency_ms": {"p50": 2076.522, "p95": 2097.586, "p99": 2097.586}, "rss_mb": 53.12},
|
||||
{"history_size": 5000, "concurrency": 4, "duration_s": 10.494, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.906, "latency_ms": {"p50": 2086.895, "p95": 2108.007, "p99": 2108.871}, "rss_mb": 42.75},
|
||||
{"history_size": 5000, "concurrency": 16, "duration_s": 10.701, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.476, "latency_ms": {"p50": 2090.597, "p95": 2183.025, "p99": 2188.819}, "rss_mb": 44.57}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
# Трек 1. Предел in-memory архитектуры — сводка. Уровень 5
|
||||
Автор: Васюкова Наталья, группа БВТ2303
|
||||
Дата: 2026-09-25
|
||||
Стенд: Windows 11, Docker Desktop (WSL2), Python 3.12, образ python:3.12-slim.
|
||||
Дата: 2026-09-25.
|
||||
Commit: solution/BVT2303-Vasyukova.
|
||||
|
||||
Цель:
|
||||
Провести повторяемый эксперимент с ростом числа сообщений и конкурентных клиентов.
|
||||
Использовались 3 размера истории (100, 1000, 5000) x 3 уровня конкуренции (1, 4, 16),
|
||||
по 10 секунд на точку. Профиль: closed-loop чтение GET /api/v1/chats/{id}/messages?limit=100.
|
||||
|
||||
Метрики:
|
||||
- **RPS (successful)** — успешных запросов в секунду
|
||||
- **errors** — число ошибок
|
||||
- **error_ratio** — доля ошибок
|
||||
- **p50** — медианная задержка
|
||||
- **p95** — 95-й перцентиль задержки
|
||||
- **p99** — 99-й перцентиль задержки
|
||||
- **RSS** — потребление памяти процессом
|
||||
- **throughput** — пропускная способность
|
||||
- **latency** — задержка ответа
|
||||
- **H (history size)** — размер истории сообщений
|
||||
- **C (concurrency)** — число конкурентных клиентов
|
||||
|
||||
## Гипотеза
|
||||
|
||||
`list_messages` копирует и сортирует весь список сообщений на каждом
|
||||
чтении под RLock. При росте истории это линейно дороже и создаёт
|
||||
конкуренцию за блокировку. Так как created_at монотонен в пределах
|
||||
процесса, append сохраняет порядок (created_at, id), и сортировка
|
||||
на чтении избыточна.
|
||||
|
||||
Изменение: убрать sort в list_messages, хранить список как есть.
|
||||
|
||||
## Ход исследования
|
||||
|
||||
Исследование проводится в 2 шага:
|
||||
|
||||
1. Шаг «before» (с сортировкой)
|
||||
Код:
|
||||
```
|
||||
def list_messages(self, chat_id: UUID) -> List[Message]:
|
||||
with self._lock:
|
||||
items = list(self._messages.get(chat_id, []))
|
||||
items.sort(key=lambda m: (m.created_at, str(m.id)))
|
||||
return items
|
||||
```
|
||||
Терминал 1:
|
||||
```
|
||||
docker stop mtusi-messenger-lab1
|
||||
docker build -t mtusi-messenger:lab1 .
|
||||
docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
|
||||
```
|
||||
Терминал 2:
|
||||
```
|
||||
python student-tests/test_load_track1.py --base-url http://localhost:8080
|
||||
Copy-Item evidence/lab1_track1_load.json evidence/lab1_track1_load_before.json
|
||||
```
|
||||
|
||||
2. Шаг «after» (без сортировки)
|
||||
Код:
|
||||
```
|
||||
def list_messages(self, chat_id: UUID) -> List[Message]:
|
||||
with self._lock:
|
||||
# Без пересортировки: список уже в порядке (created_at, id).
|
||||
return list(self._messages.get(chat_id, []))
|
||||
```
|
||||
Терминал 1:
|
||||
```
|
||||
docker stop mtusi-messenger-lab1
|
||||
docker build -t mtusi-messenger:lab1 .
|
||||
docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
|
||||
```
|
||||
Терминал 2:
|
||||
```
|
||||
python student-tests/test_load_track1.py --base-url http://localhost:8080
|
||||
Copy-Item evidence/lab1_track1_load.json evidence/lab1_track1_load_after.json
|
||||
```
|
||||
|
||||
|
||||
## Результаты
|
||||
|
||||
Во всех точках errors = 0, error_ratio = 0.0.
|
||||
|
||||
### p95, мс (before / after)
|
||||
|
||||
| H | C=1 | C=4 | C=16 |
|
||||
|-------|---------------------|---------------------|---------------------|
|
||||
| 100 | 2085.3 / 2078.9 | 2088.8 / 2090.8 | 2127.9 / 2084.3 |
|
||||
| 1000 | 2082.0 / 2076.9 | 2094.4 / 2085.5 | 2101.7 / 2093.0 |
|
||||
| 5000 | 2097.6 / 2086.7 | 2108.0 / 2065.7 | 2183.0 / 2116.6 |
|
||||
|
||||
### p99, мс (before / after)
|
||||
|
||||
| H | C=1 | C=4 | C=16 |
|
||||
|-------|---------------------|---------------------|---------------------|
|
||||
| 100 | 2085.3 / 2078.9 | 2092.0 / 2093.1 | 2134.3 / 2090.3 |
|
||||
| 1000 | 2082.0 / 2076.9 | 2106.0 / 2089.5 | 2115.5 / 2102.5 |
|
||||
| 5000 | 2097.6 / 2086.7 | 2108.9 / 2068.1 | 2188.8 / 2124.5 |
|
||||
|
||||
### RPS (успешные, before / after)
|
||||
|
||||
| H | C=1 | C=4 | C=16 |
|
||||
|-------|---------------------|---------------------|---------------------|
|
||||
| 100 | 0.483 / 0.483 | 1.920 / 1.927 | 7.555 / 7.603 |
|
||||
| 1000 | 0.482 / 0.483 | 1.927 / 1.926 | 7.562 / 7.568 |
|
||||
| 5000 | 0.481 / 0.486 | 1.906 / 1.937 | 7.476 / 7.444 |
|
||||
|
||||
### RSS, МБ (before / after)
|
||||
|
||||
| H | C=1 | C=4 | C=16 |
|
||||
|-------|---------------------|---------------------|---------------------|
|
||||
| 100 | 34.35 / 46.01 | 34.83 / 35.47 | 35.44 / 35.54 |
|
||||
| 1000 | 36.84 / 35.91 | 36.71 / 36.91 | 38.19 / 37.53 |
|
||||
| 5000 | 53.12 / 41.60 | 42.75 / 42.15 | 44.57 / 43.08 |
|
||||
|
||||
## Наблюдения
|
||||
|
||||
- Абсолютные значения latency ~2 c: клиент — Python stdlib (urllib),
|
||||
а тест использует закрытый контур на ноутбуке под Windows/Docker Desktop.
|
||||
Это ограничение клиента и среды, а не только сервера.
|
||||
- RPS растёт с конкуренцией (≈0.48 → ≈1.9 → ≈7.5) — узкое место в
|
||||
клиентском контуре, а не в API.
|
||||
- RSS растёт с историей (after: 35.5 → 37.5 → 43.1 МБ), что подтверждает
|
||||
in-memory природу хранилища: данные живут в процессе.
|
||||
- `after` ниже по RSS в большинстве точек: H=1000 — 35.9–37.5 против
|
||||
36.7–38.2; H=5000/C=4 — 42.15 против 42.75; H=5000/C=16 — 43.08
|
||||
против 44.57. Единичный выброс H=100/C=1 (46.01 после против 34.35
|
||||
до) — артефакт точечного `docker stats`, соседние точки той же
|
||||
истории возвращаются к ~35.5 МБ.
|
||||
- На H=5000/C=16 `after` лучше по latency. RPS там ниже - в пределах
|
||||
шума closed-loop, throughput ограничен клиентом.
|
||||
- На H=100 разница в пределах шума — сортировка короткого списка дёшева.
|
||||
- Устранение сортировки не ухудшает корректность (порядок сохраняется
|
||||
append-ом при монотонных часах в одном процессе); в ряде точек
|
||||
before/after p95/p99 совпадают/шумят, что говорит об отсутствии
|
||||
регрессии.
|
||||
- errors = 0 во всех точках обоих прогонов.
|
||||
|
||||
## Ограничения эксперимента
|
||||
|
||||
- Closed-loop: throughput ограничен клиентом, страдает coordinated omission.
|
||||
- Один Docker-host: HA/durability не проверяются и не заявляются.
|
||||
- Абсолютные p95/p99 не переносимы на другой ноутбук.
|
||||
- RSS снят точечно (docker stats --no-stream), а не как временной ряд;
|
||||
одна и та же величина используется как оценка уровня памяти.
|
||||
- Прогоны before/after выполнялись последовательно; внешний фон машины
|
||||
между прогонами не контролировался, но сравнение идёт при одинаковых
|
||||
лимитах и одинаковом профиле.
|
||||
|
||||
## Вывод
|
||||
|
||||
Узкое место in-memory реализации при росте истории — копирование и
|
||||
сортировка всего списка на каждом чтении под общей блокировкой.
|
||||
Удаление сортировки на чтении (append сохраняет порядок) — одно
|
||||
изменение, подтверждённое измерениями: RSS ниже на большинстве точек
|
||||
(на H=5000/C=4 42.15 против 42.75 МБ, на H=5000/C=16 43.08 против
|
||||
44.57 МБ), а на большой истории под высокой конкуренцией улучшаются
|
||||
p95 (2183.0 → 2116.6 мс) и p99 (2188.8 → 2124.5 мс) без потери
|
||||
корректности. Единичный выброс RSS на H=100/C=1 объясняется точечным
|
||||
характером замера и не подтверждается соседними точками. Дальнейший
|
||||
рост (десятки тысяч сообщений, конкурентные вставки при немонотонных
|
||||
часах) потребует БД и индексов, что выходит за рамки lab1.
|
||||
```
|
||||
|
||||
Ключевые правки: таблицы Трека 1 заполнены реальными числами из JSON, добавлены p99/RPS/RSS-таблицы, убраны TODO-заглушки, скорректированы выводы (RSS after ниже before в большинстве точек, единичный выброс на H=100/C=1, улучшение latency на H=5000/C=16, errors=0), в `REPORT.md` раздел Трека 1 приведён в соответствие с summary.
|
||||
@@ -0,0 +1,139 @@
|
||||
# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса
|
||||
|
||||
Данные: 250002 сообщений, неравномерные чаты. Запрос: `SELECT ... FROM messages WHERE chat_id = <id> AND (created_at, id) > (<середина>, …) ORDER BY created_at, id LIMIT 51`.
|
||||
Большой чат ~151818, маленький чат ~1.
|
||||
|
||||
## 1. Без индекса, большой чат
|
||||
|
||||
`Parallel Seq Scan` по всей таблице, затем `Sort`.
|
||||
|
||||
```
|
||||
Limit (cost=6593.26..6599.20 rows=51 width=70) (actual time=10.966..12.790 rows=51.00 loops=1)
|
||||
Buffers: shared hit=3210
|
||||
-> Gather Merge (cost=6593.26..11539.13 rows=42466 width=70) (actual time=10.964..12.782 rows=51.00 loops=1)
|
||||
Workers Planned: 2
|
||||
Workers Launched: 2
|
||||
Buffers: shared hit=3210
|
||||
-> Sort (cost=5593.24..5637.48 rows=17694 width=70) (actual time=7.715..7.718 rows=39.00 loops=3)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 36kB
|
||||
Buffers: shared hit=3210
|
||||
Worker 0: Sort Method: top-N heapsort Memory: 36kB
|
||||
Worker 1: Sort Method: top-N heapsort Memory: 36kB
|
||||
-> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=17694 width=70) (actual time=3.301..5.795 rows=26666.67 loops=3)
|
||||
Filter: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Rows Removed by Filter: 56667
|
||||
Buffers: shared hit=3180
|
||||
Planning:
|
||||
Buffers: shared hit=88
|
||||
Planning Time: 0.571 ms
|
||||
Execution Time: 12.922 ms
|
||||
```
|
||||
|
||||
## 2. Без индекса, маленький чат
|
||||
|
||||
Тот же `Parallel Seq Scan` по всей таблице.
|
||||
|
||||
```
|
||||
Limit (cost=6003.04..6003.05 rows=1 width=70) (actual time=7.881..10.103 rows=1.00 loops=1)
|
||||
Buffers: shared hit=3186
|
||||
-> Sort (cost=6003.04..6003.05 rows=1 width=70) (actual time=7.879..10.099 rows=1.00 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=3186
|
||||
-> Gather (cost=1000.00..6003.03 rows=1 width=70) (actual time=7.667..10.055 rows=1.00 loops=1)
|
||||
Workers Planned: 2
|
||||
Workers Launched: 2
|
||||
Buffers: shared hit=3180
|
||||
-> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=1 width=70) (actual time=3.431..4.199 rows=0.33 loops=3)
|
||||
Filter: ((chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Rows Removed by Filter: 83334
|
||||
Buffers: shared hit=3180
|
||||
Planning:
|
||||
Buffers: shared hit=88
|
||||
Planning Time: 0.662 ms
|
||||
Execution Time: 10.192 ms
|
||||
```
|
||||
|
||||
## 3. Индекс (chat_id), большой чат
|
||||
|
||||
`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`.
|
||||
|
||||
```
|
||||
Limit (cost=6594.20..6600.14 rows=51 width=70) (actual time=9.705..11.642 rows=51.00 loops=1)
|
||||
Buffers: shared hit=3210
|
||||
-> Gather Merge (cost=6594.20..11547.87 rows=42533 width=70) (actual time=9.688..11.619 rows=51.00 loops=1)
|
||||
Workers Planned: 2
|
||||
Workers Launched: 2
|
||||
Buffers: shared hit=3210
|
||||
-> Sort (cost=5594.17..5638.48 rows=17722 width=70) (actual time=6.872..6.874 rows=37.33 loops=3)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 36kB
|
||||
Buffers: shared hit=3210
|
||||
Worker 0: Sort Method: top-N heapsort Memory: 36kB
|
||||
Worker 1: Sort Method: top-N heapsort Memory: 37kB
|
||||
-> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=17722 width=70) (actual time=2.667..5.224 rows=26666.67 loops=3)
|
||||
Filter: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Rows Removed by Filter: 56667
|
||||
Buffers: shared hit=3180
|
||||
Planning:
|
||||
Buffers: shared hit=100
|
||||
Planning Time: 0.552 ms
|
||||
Execution Time: 11.745 ms
|
||||
```
|
||||
|
||||
## 4. Индекс (chat_id), маленький чат
|
||||
|
||||
`Index Scan using idx_messages_chat_only` + `Sort`.
|
||||
|
||||
```
|
||||
Limit (cost=32.15..32.16 rows=4 width=70) (actual time=0.055..0.056 rows=1.00 loops=1)
|
||||
Buffers: shared hit=10
|
||||
-> Sort (cost=32.15..32.16 rows=4 width=70) (actual time=0.054..0.055 rows=1.00 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=10
|
||||
-> Index Scan using idx_messages_chat_only on messages (cost=0.42..32.11 rows=4 width=70) (actual time=0.022..0.023 rows=1.00 loops=1)
|
||||
Index Cond: (chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid)
|
||||
Filter: (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))
|
||||
Index Searches: 1
|
||||
Buffers: shared hit=4
|
||||
Planning:
|
||||
Buffers: shared hit=100
|
||||
Planning Time: 0.642 ms
|
||||
Execution Time: 0.112 ms
|
||||
```
|
||||
|
||||
## 5. Полный индекс (chat_id, created_at, id), большой чат
|
||||
|
||||
`Index Scan using idx_messages_chat_created`, **без `Sort`**.
|
||||
|
||||
```
|
||||
Limit (cost=0.42..17.05 rows=51 width=70) (actual time=0.093..0.150 rows=51.00 loops=1)
|
||||
Buffers: shared hit=53 read=1
|
||||
-> Index Scan using idx_messages_chat_created on messages (cost=0.42..13689.96 rows=41992 width=70) (actual time=0.092..0.146 rows=51.00 loops=1)
|
||||
Index Cond: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Index Searches: 1
|
||||
Buffers: shared hit=53 read=1
|
||||
Planning:
|
||||
Buffers: shared hit=111
|
||||
Planning Time: 0.751 ms
|
||||
Execution Time: 0.208 ms
|
||||
```
|
||||
|
||||
## 6. Полный индекс (chat_id, created_at, id), маленький чат
|
||||
|
||||
`Index Scan` без `Sort`.
|
||||
|
||||
```
|
||||
Limit (cost=0.42..8.44 rows=1 width=70) (actual time=0.028..0.029 rows=1.00 loops=1)
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using idx_messages_chat_created on messages (cost=0.42..8.44 rows=1 width=70) (actual time=0.027..0.028 rows=1.00 loops=1)
|
||||
Index Cond: ((chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Index Searches: 1
|
||||
Buffers: shared hit=4
|
||||
Planning:
|
||||
Buffers: shared hit=111
|
||||
Planning Time: 0.846 ms
|
||||
Execution Time: 0.105 ms
|
||||
```
|
||||
@@ -0,0 +1,8 @@
|
||||
Track 2 restore
|
||||
dump_time_s=1.701
|
||||
dump_size_bytes=7783813
|
||||
restore_time_s=1.623
|
||||
count_original=280002
|
||||
count_restored=280002
|
||||
dist_match=true
|
||||
random100_match=true
|
||||
@@ -0,0 +1,4 @@
|
||||
Track 2 RPO
|
||||
marker_message_id=9c0393a4-4fbd-4c79-8229-1fdd065eb7d8
|
||||
present_in_restore=0
|
||||
expected=0
|
||||
@@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"variant": "no_index",
|
||||
"insert_10000_s": 0.873,
|
||||
"indexes_size": "10 MB",
|
||||
"total_size": "36 MB"
|
||||
},
|
||||
{
|
||||
"variant": "chat_only",
|
||||
"insert_10000_s": 0.888,
|
||||
"indexes_size": "12 MB",
|
||||
"total_size": "39 MB"
|
||||
},
|
||||
{
|
||||
"variant": "full",
|
||||
"insert_10000_s": 0.805,
|
||||
"indexes_size": "26 MB",
|
||||
"total_size": "54 MB"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: 1
|
||||
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: messenger-local
|
||||
receivers:
|
||||
- uid: messenger-local-webhook
|
||||
type: webhook
|
||||
disableResolveMessage: false
|
||||
settings:
|
||||
# Доставка наружу не требуется; локальный no-op endpoint.
|
||||
url: http://127.0.0.1:9/messenger-alerts
|
||||
httpMethod: POST
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: messenger-local
|
||||
group_by: ['alertname']
|
||||
group_wait: 10s
|
||||
group_interval: 1m
|
||||
repeat_interval: 4h
|
||||
@@ -0,0 +1,109 @@
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: messenger-alerts
|
||||
folder: Messenger
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: messenger-high-error-rate
|
||||
title: MessengerHighErrorRate
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 60
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
editorMode: code
|
||||
expr: sum(rate(http_requests_total{http_status_class="5xx"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])), 1e-9)
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params: [0.05]
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: [C]
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
for: 30s
|
||||
annotations:
|
||||
summary: "API error rate > 5% over 1m"
|
||||
description: "5xx ratio exceeds SLO 5% for 30 seconds."
|
||||
runbook: "infra/runbooks/high-error-rate.md"
|
||||
labels:
|
||||
severity: warning
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
isPaused: false
|
||||
|
||||
- uid: messenger-jobs-backlog-growing
|
||||
title: MessengerJobsBacklogGrowing
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
editorMode: code
|
||||
expr: sum(increase(jobs_processed_total{outcome="retry"}[10m])) - sum(increase(jobs_processed_total{outcome="success"}[10m]))
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params: [5]
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: [C]
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Retries outpace successes"
|
||||
description: "Worker retry count exceeds successes by >5 for 5m."
|
||||
runbook: "infra/runbooks/queue-lag.md"
|
||||
labels:
|
||||
severity: warning
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
isPaused: false
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger — lab6 observability",
|
||||
"schemaVersion": 39,
|
||||
"version": 2,
|
||||
"editable": true,
|
||||
"timezone": "browser",
|
||||
"time": {"from": "now-30m", "to": "now"},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "service",
|
||||
"type": "query",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"query": "label_values(up, job)",
|
||||
"current": {"text": "otel-collector", "value": "otel-collector"}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API RPS (total)",
|
||||
"gridPos": {"x": 0, "y": 0, "w": 8, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API error ratio (5xx / total)",
|
||||
"gridPos": {"x": 8, "y": 0, "w": 8, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP latency p50/p95/p99 (s)",
|
||||
"gridPos": {"x": 16, "y": 0, "w": 8, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Jobs processed by outcome",
|
||||
"gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Job duration p95 (s) by outcome",
|
||||
"gridPos": {"x": 12, "y": 8, "w": 12, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "logs",
|
||||
"title": "API + worker logs",
|
||||
"gridPos": {"x": 0, "y": 16, "w": 24, "h": 10},
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"targets": [
|
||||
{"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}", "refId": "A"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards/json
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: 'trace_id=([a-f0-9]+)'
|
||||
datasourceUid: tempo
|
||||
url: "$${__value.raw}"
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service_name
|
||||
filterByTraceID: true
|
||||
@@ -0,0 +1,42 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
instance_addr: 127.0.0.1
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
retention_period: 168h
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
allow_structured_metadata: true
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
delete_request_store: filesystem
|
||||
retention_enabled: true
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,53 @@
|
||||
# lab7: два server-блока:
|
||||
# :8443 — внешний HTTPS (server cert), проксирует в upstream по mTLS
|
||||
# :8080 — plain на loopback? Нет: API-реплики слушают TLS, LB ходит к ним по mTLS.
|
||||
# resolver и proxy_pass по имени api (Docker DNS) без upstream{}.
|
||||
|
||||
resolver 127.0.0.11 valid=5s ipv6=off;
|
||||
|
||||
server {
|
||||
# внешний TLS ingress
|
||||
listen 8443 ssl;
|
||||
server_name localhost;
|
||||
|
||||
ssl_certificate /etc/nginx/certs/server.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/server.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_prefer_server_ciphers on;
|
||||
|
||||
# API-реплики предъявляют server cert, подписанный нашим CA.
|
||||
# LB обязан предъявить клиентский сертификат — это client cert LB.
|
||||
ssl_client_certificate /etc/nginx/certs/ca.crt;
|
||||
|
||||
client_max_body_size 10m;
|
||||
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
|
||||
location / {
|
||||
set $api_upstream "https://api:8080";
|
||||
proxy_pass $api_upstream;
|
||||
|
||||
# mTLS к API: LB аутентифицируется своим client cert.
|
||||
proxy_ssl_certificate /etc/nginx/certs/lb-client.crt;
|
||||
proxy_ssl_certificate_key /etc/nginx/certs/lb-client.key;
|
||||
proxy_ssl_trusted_certificate /etc/nginx/certs/ca.crt;
|
||||
proxy_ssl_verify on;
|
||||
proxy_ssl_verify_depth 2;
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name api;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location = /lb/health {
|
||||
return 200 "ok\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# API читает только свой secret/path.
|
||||
path "secret/data/app/api" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
path "secret/metadata/app/api" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# OpenBao non-dev: file storage, TLS listener.
|
||||
# disable_mlock перенесён в блок storage, так как на верхнем уровне
|
||||
# OpenBao его не распознаёт и выдаёт предупреждение.
|
||||
ui = false
|
||||
|
||||
storage "file" {
|
||||
path = "/openbao/data"
|
||||
# Отключаем mlock, чтобы процесс корректно завершался и освобождал порт.
|
||||
disable_mlock = true
|
||||
}
|
||||
|
||||
listener "tcp" {
|
||||
# lab7 fix: сменили 8201 → 8200. На Windows-host Docker Desktop остаётся
|
||||
# "зомби"-listener на 8201 (wslrelay/wslhost), невидимый ни в docker ps,
|
||||
# ни в netstat внутри WSL2. Порт внутренний, наружу не публикуется —
|
||||
# контракт лабораторной не нарушается.
|
||||
address = "0.0.0.0:8200"
|
||||
tls_disable = false
|
||||
tls_cert_file = "/certs/bao-server.crt"
|
||||
tls_key_file = "/certs/bao-server.key"
|
||||
tls_client_ca_file = "/certs/ca.crt"
|
||||
}
|
||||
|
||||
api_addr = "https://openbao:8200"
|
||||
# cluster_addr убран: одиночный узел, порт 8202 не слушается.
|
||||
@@ -0,0 +1,7 @@
|
||||
# Worker читает только свой secret/path. НЕ получает admin БД/S3/API.
|
||||
path "secret/data/app/worker" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
path "secret/metadata/app/worker" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
tls:
|
||||
cert_file: /certs/collector-server.crt
|
||||
key_file: /certs/collector-server.key
|
||||
client_ca_file: /certs/ca.crt
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
tls:
|
||||
cert_file: /certs/collector-server.crt
|
||||
key_file: /certs/collector-server.key
|
||||
client_ca_file: /certs/ca.crt
|
||||
|
||||
processors:
|
||||
# Level-4: ограничение RAM и batch, чтобы не расти бесконечно при
|
||||
# недоступном backend и уменьшить объём записи.
|
||||
memory_limiter:
|
||||
check_interval: 2s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 64
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
resource:
|
||||
attributes:
|
||||
- key: deployment.environment.name
|
||||
value: lab
|
||||
action: upsert
|
||||
|
||||
exporters:
|
||||
# Явно ограничиваем retry и время ожидания, чтобы при недоступном
|
||||
# бэкенде коллектор не накапливал данные и не падал.
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
tls:
|
||||
insecure: true
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 4
|
||||
queue_size: 1000
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
tls:
|
||||
insecure: true
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 4
|
||||
queue_size: 1000
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
send_timestamps: true
|
||||
metric_expiration: 5m
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
address: 0.0.0.0:8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,12 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yaml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
@@ -0,0 +1,44 @@
|
||||
groups:
|
||||
- name: messenger-alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{http_status_class="5xx"}[1m]))
|
||||
/
|
||||
sum(rate(http_requests_total[1m]))
|
||||
> 0.05
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "API error rate > 5% over 1m"
|
||||
description: "5xx ratio exceeds SLO 5% for 30 seconds."
|
||||
runbook: "infra/runbooks/high-error-rate.md"
|
||||
|
||||
- alert: MessengerJobsBacklogGrowing
|
||||
expr: |
|
||||
sum(increase(jobs_processed_total{outcome="retry"}[10m]))
|
||||
>
|
||||
sum(increase(jobs_processed_total{outcome="success"}[10m]))
|
||||
+ 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Retries outpace successes"
|
||||
description: "Worker retry count exceeds successes for 5m."
|
||||
runbook: "infra/runbooks/queue-lag.md"
|
||||
|
||||
- alert: MessengerUploadLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (le, http_route)
|
||||
(rate(http_request_duration_seconds_bucket{http_route=~"/api/v1/chats/.*/attachments"}[5m]))
|
||||
) > 2
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "p95 upload latency > 2s"
|
||||
runbook: "infra/runbooks/slow-upload.md"
|
||||
@@ -0,0 +1,19 @@
|
||||
# Alert: MessengerHighErrorRate
|
||||
|
||||
## Симптомы
|
||||
- 5xx rate > 5% в течение 2 минут.
|
||||
- Панель «API error ratio» в Grafana.
|
||||
|
||||
## Диагностика (по шагам)
|
||||
1. Открыть Grafana → Dashboard «Messenger» → p95 latency.
|
||||
2. Открыть Explore → Loki: `{service_name="messenger-api"} |= "event=unhandled_exception"`.
|
||||
3. Найти в логах `request_id`, затем в Tempo trace с этим span или по
|
||||
trace_id из поля `trace_id`.
|
||||
4. Смотрим какие зависимость отвечает: postgres/redis/s3. Соответствующие
|
||||
ошибки пишутся как `event=db_unavailable`, `event=redis_unavailable`,
|
||||
`event=s3_unavailable`.
|
||||
|
||||
## Действие
|
||||
- Если виноват backend — перезапустить (`docker compose restart <service>`).
|
||||
- Если ошибка идёт от кода — rollback через `docker compose up -d --build`
|
||||
предыдущего тега.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Alert: MessengerJobsBacklogGrowing
|
||||
|
||||
## Симптомы
|
||||
- `jobs_processed_total{outcome="retry"}` > `success` + 5 за 10 минут.
|
||||
|
||||
## Диагностика
|
||||
1. Проверить `docker compose ps worker` — процесс жив?
|
||||
2. `docker compose exec rabbitmq rabbitmqctl list_queues name messages consumers`
|
||||
3. Открыть Grafana → «Jobs processed by outcome» и «Job duration p95».
|
||||
4. Открыть Tempo, найти span `job.consume` с ошибкой (status=ERROR).
|
||||
|
||||
## Действие
|
||||
- Если worker упал — `docker compose restart worker`.
|
||||
- Если очередь переполнена по `retry_exhausted` — проверить S3 (`minio`
|
||||
контейнер healthy), затем `docker compose restart s3_cleanup`.
|
||||
- При долгом backlog включить alert `for: 10m` и снизить `MAX_ATTEMPTS` в `.env`
|
||||
для быстрого drain.
|
||||
@@ -0,0 +1,15 @@
|
||||
# Alert: MessengerUploadLatencyHigh
|
||||
|
||||
## Симптомы
|
||||
- p95 длительности POST `/api/v1/chats/{id}/attachments` > 2s.
|
||||
|
||||
## Диагностика
|
||||
1. Grafana: p95 по route `/api/v1/chats/{id}/attachments`.
|
||||
2. Tempo: trace `POST /api/v1/chats/{id}/attachments` — найти самый долгий
|
||||
child span (`put_object` в S3 или `INSERT` в postgres).
|
||||
3. Loki: `{service_name="messenger-api"} |= "event=http" |~ "attachments"`.
|
||||
|
||||
## Действие
|
||||
- Если виноват S3 — проверить `docker compose ps minio`.
|
||||
- Если БД — увеличить `max_size` пула или `statement_timeout`.
|
||||
- Если телеметрия — переключить sampling (см. TRACK1).
|
||||
@@ -0,0 +1,34 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
grpc_listen_port: 9095
|
||||
log_level: warn
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
|
||||
ingester:
|
||||
trace_idle_period: 10s
|
||||
max_block_bytes: 1048576
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
|
||||
metrics_generator:
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
@@ -0,0 +1,14 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.32.0
|
||||
psycopg[binary]==3.2.3
|
||||
psycopg_pool==3.2.3
|
||||
redis==5.2.0
|
||||
boto3==1.35.74
|
||||
Pillow==11.0.0
|
||||
pika==1.3.2
|
||||
argon2-cffi==23.1.0
|
||||
opentelemetry-api==1.31.0
|
||||
opentelemetry-sdk==1.31.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.31.0
|
||||
opentelemetry-instrumentation-fastapi==0.52b0
|
||||
opentelemetry-instrumentation-requests==0.52b0
|
||||
@@ -0,0 +1,33 @@
|
||||
@echo off
|
||||
REM lab7: Windows-эквивалент scripts/restart.sh.
|
||||
REM Перезапускает LB, обе реплики API, PostgreSQL, Redis, MinIO, s3_cleanup.
|
||||
REM OpenBao после рестарта sealed — unseal делает openbao-init (идемпотентно).
|
||||
REM Volumes сохраняются; схема/данные не теряются.
|
||||
|
||||
setlocal enableextensions enabledelayedexpansion
|
||||
|
||||
REM 1) OpenBao: unseal через one-shot openbao-init.
|
||||
docker compose --env-file .env up -d openbao >nul 2>&1
|
||||
docker compose --env-file .env run --rm --no-deps openbao-init >nul 2>&1
|
||||
|
||||
REM 2) Перезапуск остальных сервисов.
|
||||
docker compose --env-file .env restart lb api postgres redis minio s3_cleanup >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
docker restart mtuci-lb >nul 2>&1
|
||||
docker restart mtuci-postgres >nul 2>&1
|
||||
docker restart mtuci-redis >nul 2>&1
|
||||
docker restart mtuci-minio >nul 2>&1
|
||||
for /f "tokens=*" %%c in ('docker ps -a --filter "name=mtuci2026lab7_api" -q') do (
|
||||
docker restart %%c >nul 2>&1
|
||||
)
|
||||
)
|
||||
|
||||
REM 3) Ждём HTTPS readiness через LB с проверкой CA (до 90 секунд).
|
||||
set /a i=0
|
||||
:wait_loop
|
||||
if %i% geq 90 exit /b 1
|
||||
curl -fsS --cacert .\certs\ca.crt https://localhost:8443/health/ready >nul 2>&1
|
||||
if not errorlevel 1 exit /b 0
|
||||
set /a i+=1
|
||||
timeout /t 1 /nobreak >nul
|
||||
goto wait_loop
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# lab7: перезапуск LB, обеих реплик API, PostgreSQL, Redis, MinIO, OpenBao
|
||||
# и s3_cleanup. Volumes сохраняются. После рестарта OpenBao sealed —
|
||||
# его надо распечатать (unseal), что делает openbao-init (идемпотентно).
|
||||
|
||||
# 1) openbao после рестарта sealed — unseal через openbao-init.
|
||||
# openbao-init зависит от openbao (healthy) и security_init (completed).
|
||||
docker compose --env-file .env up -d openbao >/dev/null 2>&1 || true
|
||||
docker compose --env-file .env run --rm --no-deps openbao-init >/dev/null 2>&1 || true
|
||||
|
||||
# 2) Перезапуск остальных сервисов (volumes сохраняются).
|
||||
docker compose --env-file .env restart lb api postgres redis minio s3_cleanup >/dev/null 2>&1 || {
|
||||
docker restart mtuci-lb >/dev/null 2>&1 || true
|
||||
docker restart mtuci-postgres >/dev/null 2>&1 || true
|
||||
docker restart mtuci-redis >/dev/null 2>&1 || true
|
||||
docker restart mtuci-minio >/dev/null 2>&1 || true
|
||||
for c in $(docker ps -a --filter "name=mtuci2026lab7_api" -q); do
|
||||
docker restart "$c" >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
# 3) Ждём готовности LB :8443 по HTTPS с проверкой CA.
|
||||
i=0
|
||||
while [ $i -lt 90 ]; do
|
||||
if curl -fsS --cacert ./certs/ca.crt https://localhost:8443/health/ready >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
i=$((i+1)); sleep 1
|
||||
done
|
||||
exit 1
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
# Поочерёдное обновление API-реплик под нагрузкой.
|
||||
# Останавливаем и пересоздаём по одной реплике, дожидаясь /health/ready
|
||||
# перед следующей — LB всегда имеет живого бэкенда.
|
||||
set -eu
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
API_SERVICE="api"
|
||||
REPLICAS=2
|
||||
READY_TIMEOUT=60
|
||||
|
||||
wait_ready() {
|
||||
i=0
|
||||
while [ "$i" -lt "$READY_TIMEOUT" ]; do
|
||||
if curl -fsS "$BASE_URL/health/ready" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "health/ready не поднялся за ${READY_TIMEOUT}s" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Собираем новый образ один раз, до остановки реплик.
|
||||
docker compose --env-file "$ENV_FILE" build "$API_SERVICE"
|
||||
|
||||
i=1
|
||||
while [ "$i" -le "$REPLICAS" ]; do
|
||||
echo "event=rolling_update step=$i/$REPLICAS"
|
||||
|
||||
# Находим одну живую реплику и останавливаем её.
|
||||
cid=$(docker ps --filter "label=com.docker.compose.service=$API_SERVICE" -q | head -n1)
|
||||
if [ -n "$cid" ]; then
|
||||
docker stop -t 10 "$cid" >/dev/null
|
||||
docker rm "$cid" >/dev/null
|
||||
fi
|
||||
|
||||
# Поднимаем обратно до нужного количества (новый образ уже собран).
|
||||
docker compose --env-file "$ENV_FILE" up -d --no-deps --scale "$API_SERVICE=$REPLICAS" "$API_SERVICE" >/dev/null
|
||||
|
||||
wait_ready
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
echo "event=rolling_update_done"
|
||||
@@ -0,0 +1,39 @@
|
||||
@echo off
|
||||
REM Windows-эквивалент scripts/stop-one.sh для lab3.
|
||||
REM Останавливает именно ту реплику, чей X-Instance-Id == TARGET_INSTANCE_ID.
|
||||
REM DB/LB/volumes не трогаем; восстановление — docker compose up -d api.
|
||||
|
||||
setlocal enabledelayedexpansion
|
||||
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID not set >&2
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set TARGET_ID=
|
||||
for /f "tokens=*" %%c in ('docker ps --filter "label=com.docker.compose.service=api" -q') do (
|
||||
for /f "tokens=*" %%v in ('docker exec %%c cat /tmp/instance_id 2^>nul') do (
|
||||
if "%%v"=="%TARGET_INSTANCE_ID%" (
|
||||
set TARGET_ID=%%c
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if "!TARGET_ID!"=="" (
|
||||
echo no api replica with INSTANCE_ID=%TARGET_INSTANCE_ID% >&2
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Stopping api replica !TARGET_ID! (INSTANCE_ID=%TARGET_INSTANCE_ID%)
|
||||
docker stop -t 3 !TARGET_ID! >nul
|
||||
|
||||
REM Ждём, пока контейнер реально уйдёт из Up (максимум ~5 секунд).
|
||||
set /a i=0
|
||||
:wait_loop
|
||||
if %i% geq 50 exit /b 0
|
||||
for /f "tokens=*" %%c in ('docker ps --filter "id=!TARGET_ID!" --format "{{.ID}}"') do (
|
||||
timeout /t 1 /nobreak >nul
|
||||
set /a i+=1
|
||||
goto wait_loop
|
||||
)
|
||||
exit /b 0
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
TARGET=""
|
||||
for c in $(docker ps --filter "label=com.docker.compose.service=api" -q); do
|
||||
id=$(docker exec "$c" cat /tmp/instance_id 2>/dev/null || true)
|
||||
if [ "$id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
TARGET="$c"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$TARGET" ]; then
|
||||
echo "no api replica with INSTANCE_ID=$TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker stop -t 3 "$TARGET" >/dev/null
|
||||
|
||||
i=0
|
||||
while [ $i -lt 50 ]; do
|
||||
if ! docker ps --filter "id=$TARGET" --format "{{.ID}}" | grep -q .; then
|
||||
exit 0
|
||||
fi
|
||||
i=$((i+1)); sleep 0.1
|
||||
done
|
||||
exit 0
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Вложения (lab5/lab6): upload → 202 queued, async worker, download вариантов.
|
||||
|
||||
lab6: при создании outbox-строки кладём в payload W3C traceparent/tracestate
|
||||
текущего HTTP-запроса. Ошибка инжекта не должна блокировать upload.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import unicodedata
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import Response
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from botocore.exceptions import ClientError
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
from ..models.schemas import Attachment, User
|
||||
from ..storage import s3
|
||||
from ..storage.postgres import storage
|
||||
from .security import current_user
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["attachments"])
|
||||
|
||||
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
PIPELINE_VERSION = 1
|
||||
_propagator = TraceContextTextMapPropagator()
|
||||
|
||||
|
||||
def _require_member(chat_id: UUID, user_id: UUID):
|
||||
chat = storage.get_chat(chat_id)
|
||||
if chat is None or user_id not in chat.member_ids:
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
return chat
|
||||
|
||||
|
||||
def _validate_filename(name: str) -> str:
|
||||
if not name or len(name) > 255:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
if name.strip() != name or name.strip() == "":
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
for ch in name:
|
||||
if ch in ("/", "\\", "\x00") or unicodedata.category(ch).startswith("C"):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
return name
|
||||
|
||||
|
||||
def _detect_content_type(data: bytes) -> Optional[str]:
|
||||
"""MIME по содержимому, не по заголовку."""
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def _validate_image(data: bytes) -> str:
|
||||
"""MIME по содержимому + лимит пикселей + полная проверка декодирования."""
|
||||
ctype = _detect_content_type(data)
|
||||
if ctype is None:
|
||||
raise HTTPException(status_code=415, detail="unsupported_media_type")
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
width, height = img.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise HTTPException(status_code=415, detail="unsupported_media_type")
|
||||
if width * height > MAX_IMAGE_PIXELS:
|
||||
raise HTTPException(status_code=413, detail="payload_too_large")
|
||||
img.load()
|
||||
except HTTPException:
|
||||
raise
|
||||
except Image.DecompressionBombError:
|
||||
raise HTTPException(status_code=413, detail="payload_too_large")
|
||||
except Exception:
|
||||
raise HTTPException(status_code=415, detail="unsupported_media_type")
|
||||
return ctype
|
||||
|
||||
|
||||
@router.post(
|
||||
path="/chats/{chat_id}/attachments",
|
||||
summary="Принять один PNG/JPEG, до 10 MiB (lab5: 202 queued)",
|
||||
response_model=Attachment,
|
||||
status_code=202,
|
||||
)
|
||||
async def upload_attachment(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
user: User = Depends(current_user),
|
||||
filename: str = Query(...),
|
||||
) -> Attachment:
|
||||
_require_member(chat_id, user.id)
|
||||
safe_name = _validate_filename(filename)
|
||||
|
||||
cl = request.headers.get("content-length")
|
||||
if cl is not None:
|
||||
try:
|
||||
if int(cl) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(status_code=413, detail="payload_too_large")
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(status_code=413, detail="payload_too_large")
|
||||
chunks.append(chunk)
|
||||
body = b"".join(chunks)
|
||||
if not body:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
content_type = await run_in_threadpool(_validate_image, body)
|
||||
sha256 = hashlib.sha256(body).hexdigest()
|
||||
|
||||
attachment_id = uuid4()
|
||||
staging_key = s3.staging_key(chat_id, attachment_id)
|
||||
|
||||
# 1) Durable staging в S3 (приватный prefix).
|
||||
try:
|
||||
await run_in_threadpool(s3.put_bytes, staging_key, body, content_type)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=503, detail="unavailable") from exc
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
att = Attachment(
|
||||
id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user.id,
|
||||
filename=safe_name,
|
||||
content_type=content_type,
|
||||
size_bytes=len(body),
|
||||
sha256=sha256,
|
||||
status="queued",
|
||||
variants=[],
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
job_id = uuid4()
|
||||
|
||||
# lab6: снимок W3C trace context активного HTTP span.
|
||||
trace_carrier: dict[str, str] = {}
|
||||
try:
|
||||
_propagator.inject(trace_carrier)
|
||||
except Exception:
|
||||
trace_carrier = {}
|
||||
|
||||
outbox_payload = {
|
||||
"job_id": str(job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": PIPELINE_VERSION,
|
||||
"created_at": now.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
if trace_carrier.get("traceparent"):
|
||||
outbox_payload["traceparent"] = trace_carrier["traceparent"]
|
||||
if trace_carrier.get("tracestate"):
|
||||
outbox_payload["tracestate"] = trace_carrier["tracestate"]
|
||||
|
||||
# 2) INSERT attachment + outbox в ОДНОЙ транзакции.
|
||||
# При сбое — компенсация: удаляем staging.
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
storage.create_attachment_queued, att, staging_key, outbox_payload
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
await run_in_threadpool(s3.delete_object, staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
raise HTTPException(status_code=503, detail="unavailable")
|
||||
|
||||
return att
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/attachments/{attachment_id}",
|
||||
summary="Метаданные вложения, доступ участникам чата",
|
||||
response_model=Attachment,
|
||||
)
|
||||
def get_attachment(
|
||||
attachment_id: UUID,
|
||||
user: User = Depends(current_user),
|
||||
) -> Attachment:
|
||||
pair = storage.get_attachment_with_key(attachment_id)
|
||||
if pair is None:
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
att, _key = pair
|
||||
_require_member(att.chat_id, user.id)
|
||||
return att
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/attachments/{attachment_id}/content",
|
||||
summary="Скачать байты (original или variant) через авторизованный API",
|
||||
)
|
||||
def download_attachment(
|
||||
attachment_id: UUID,
|
||||
user: User = Depends(current_user),
|
||||
variant: str = Query("original"),
|
||||
):
|
||||
if variant not in ("original", "thumbnail", "crop", "optimized"):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
pair = storage.get_attachment_with_key(attachment_id)
|
||||
if pair is None:
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
att, object_key = pair
|
||||
_require_member(att.chat_id, user.id)
|
||||
if att.status != "ready":
|
||||
raise HTTPException(status_code=409, detail="conflict")
|
||||
|
||||
if variant == "original":
|
||||
key = object_key
|
||||
ctype_expected = att.content_type
|
||||
else:
|
||||
vs = [v for v in att.variants if v.name == variant]
|
||||
if not vs:
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
v = vs[0]
|
||||
key = s3.variant_key(att.chat_id, att.id, variant)
|
||||
ctype_expected = v.content_type
|
||||
|
||||
try:
|
||||
data, ctype = s3.get_object(key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code in ("404", "NoSuchKey", "NotFound"):
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
raise HTTPException(status_code=503, detail="unavailable") from exc
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=503, detail="unavailable") from exc
|
||||
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=ctype or ctype_expected,
|
||||
)
|
||||
@@ -0,0 +1,122 @@
|
||||
import base64
|
||||
import binascii
|
||||
import os
|
||||
import time
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from ..models.schemas import Session
|
||||
from ..storage.postgres import storage
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
|
||||
|
||||
DEFAULT_TTL_SECONDS = 3600
|
||||
|
||||
# --- lab7: ограничение частоты входов (общее для реплик через Redis) ---
|
||||
LOGIN_WINDOW_SECONDS = int(os.environ.get("LOGIN_WINDOW_SECONDS", "60"))
|
||||
LOGIN_MAX_ATTEMPTS = int(os.environ.get("LOGIN_MAX_ATTEMPTS", "10"))
|
||||
|
||||
|
||||
def _ttl_seconds() -> int:
|
||||
try:
|
||||
return int(os.environ.get("SESSION_TTL_SECONDS", DEFAULT_TTL_SECONDS))
|
||||
except ValueError:
|
||||
return DEFAULT_TTL_SECONDS
|
||||
|
||||
|
||||
def _parse_basic_header(request: Request) -> tuple[str, str] | None:
|
||||
"""Ручной разбор Basic: base64 -> UTF-8 -> split по первому ':'.
|
||||
|
||||
HTTPBasic из Starlette декодирует latin-1 и ломает кириллицу.
|
||||
Здесь пароль читается как UTF-8 (RFC 7617 разрешает UTF-8 charset).
|
||||
Username остаётся ASCII по контракту.
|
||||
"""
|
||||
raw = request.headers.get("Authorization", "")
|
||||
if not raw.lower().startswith("basic "):
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(raw[6:].strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
return None
|
||||
if ":" not in decoded:
|
||||
return None
|
||||
username, password = decoded.split(":", 1)
|
||||
if not username or not password:
|
||||
return None
|
||||
try:
|
||||
username.encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def _login_rate_limit_key(request: Request) -> str:
|
||||
"""Ключ лимита: username + IP клиента (за LB — X-Forwarded-For)."""
|
||||
parsed = _parse_basic_header(request)
|
||||
username = parsed[0] if parsed else "-"
|
||||
fwd = request.headers.get("X-Forwarded-For", "")
|
||||
ip = fwd.split(",")[0].strip() if fwd else (request.client.host if request.client else "-")
|
||||
return "loginrl:%s:%s" % (username, ip)
|
||||
|
||||
|
||||
def _check_login_rate_limit(request: Request) -> None:
|
||||
"""Счётчик попыток в Redis. При превышении — 429 с Retry-After."""
|
||||
try:
|
||||
from ..storage.sessions import _redis_client
|
||||
r = _redis_client()
|
||||
except RuntimeError:
|
||||
# Redis недоступен — полагаемся на доступность БД; не блокируем вход.
|
||||
return
|
||||
key = _login_rate_limit_key(request)
|
||||
try:
|
||||
current = r.incr(key)
|
||||
if current == 1:
|
||||
r.expire(key, LOGIN_WINDOW_SECONDS)
|
||||
if current > LOGIN_MAX_ATTEMPTS:
|
||||
ttl = r.ttl(key)
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="rate_limited",
|
||||
headers={"Retry-After": str(max(1, int(ttl) if ttl and ttl > 0 else LOGIN_WINDOW_SECONDS))},
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
# Ошибка Redis в счётчике не должна блокировать вход целиком.
|
||||
return
|
||||
|
||||
|
||||
@router.post(
|
||||
path="/sessions",
|
||||
summary="Вход: Basic username:password; выдаёт сессию",
|
||||
response_model=Session,
|
||||
status_code=201,
|
||||
)
|
||||
def create_session(request: Request) -> Session:
|
||||
_check_login_rate_limit(request)
|
||||
parsed = _parse_basic_header(request)
|
||||
if parsed is None:
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
username, password = parsed
|
||||
user = storage.verify_credentials(username, password)
|
||||
if user is None:
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
token, expires_at = storage.create_session(user.id, _ttl_seconds())
|
||||
return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user)
|
||||
|
||||
|
||||
@router.delete(
|
||||
path="/sessions/current",
|
||||
summary="Отозвать текущую сессию",
|
||||
status_code=204,
|
||||
)
|
||||
def revoke_session(request: Request) -> None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if not auth.lower().startswith("bearer "):
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
token = auth[7:].strip()
|
||||
if not token or storage.get_session_user(token) is None:
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
storage.revoke_session(token)
|
||||
return None
|
||||
@@ -0,0 +1,94 @@
|
||||
import base64
|
||||
import json
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from ..models.schemas import Chat, ChatPage, CreateChat, User
|
||||
from ..storage.postgres import storage
|
||||
from .security import current_user
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
|
||||
def _encode_cursor(user_id: UUID, chat: Chat) -> str:
|
||||
"""Простой курсор: user_id + id + created_at, base64."""
|
||||
data = {"user_id": str(user_id), "id": str(chat.id), "created_at": chat.created_at.isoformat()}
|
||||
return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(user_id: UUID, cursor: str) -> tuple[datetime, UUID]:
|
||||
"""Курсор привязан к пользователю. Любая порча → 400, не 500."""
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
if not isinstance(data, dict):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
if data.get("user_id") != str(user_id):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
try:
|
||||
created_at = datetime.fromisoformat(data["created_at"])
|
||||
chat_id = UUID(data["id"])
|
||||
except (KeyError, ValueError, TypeError):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
return created_at, chat_id
|
||||
|
||||
|
||||
@router.post(
|
||||
path="",
|
||||
summary="Создать чат",
|
||||
response_model=Chat,
|
||||
status_code=201,
|
||||
)
|
||||
def create_chat(
|
||||
payload: CreateChat,
|
||||
user: User = Depends(current_user),
|
||||
) -> Chat:
|
||||
"""Вызывающий должен входить в member_ids"""
|
||||
user_id = user.id
|
||||
if user_id not in payload.member_ids:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
if len(set(payload.member_ids)) != len(payload.member_ids):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
try:
|
||||
return storage.create_chat(payload.title, list(payload.member_ids))
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
|
||||
@router.get(
|
||||
path="",
|
||||
summary="Получить список чатов текущего пользователя",
|
||||
response_model=ChatPage,
|
||||
)
|
||||
def list_chats(
|
||||
user: User = Depends(current_user),
|
||||
limit: int = Query(50, ge=1, le=100),
|
||||
cursor: str | None = None,
|
||||
) -> ChatPage:
|
||||
user_id = user.id
|
||||
chats = storage.list_chats_for_user(user_id)
|
||||
if cursor:
|
||||
ccreated, cid = _decode_cursor(user_id, cursor)
|
||||
ccreated_iso = ccreated.isoformat()
|
||||
chats = [c for c in chats if (c.created_at.isoformat(), str(c.id)) > (ccreated_iso, str(cid))]
|
||||
page = chats[:limit]
|
||||
next_cursor = _encode_cursor(user_id, page[-1]) if len(chats) > limit else None
|
||||
return ChatPage(items=page, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/{chat_id}",
|
||||
summary="Получить доступный чат по идентификатору",
|
||||
response_model=Chat,
|
||||
)
|
||||
def get_chat(
|
||||
chat_id: UUID,
|
||||
user: User = Depends(current_user),
|
||||
) -> Chat:
|
||||
chat = storage.get_chat(chat_id)
|
||||
if chat is None or user.id not in chat.member_ids:
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
return chat
|
||||
@@ -0,0 +1,52 @@
|
||||
from fastapi import APIRouter
|
||||
from ..models.schemas import Health
|
||||
from ..storage import db as _db
|
||||
from ..storage import s3 as _s3
|
||||
from ..storage.sessions import _redis_client, _store
|
||||
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/health/live",
|
||||
summary="Проверка жизнеспособности процесса",
|
||||
response_model=Health,
|
||||
)
|
||||
def liveness() -> Health:
|
||||
"""Процесс жив"""
|
||||
return Health(status="ok")
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/health/ready",
|
||||
summary="Проверка готовности принимать трафик",
|
||||
response_model=Health,
|
||||
)
|
||||
def readiness() -> Health:
|
||||
"""200, если PostgreSQL, Redis (в режиме redis) и S3 доступны.
|
||||
|
||||
При недоступности зависимости выбрасываем RuntimeError("*_unavailable"),
|
||||
main.py конвертирует его в 503 с телом Error из contracts/openapi.json.
|
||||
Все проверки — с короткими таймаутами, чтобы не зависать надолго.
|
||||
"""
|
||||
try:
|
||||
with _db.connection() as conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
cur.fetchone()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Пул уже проверяет живость соединения (check_connection) и короткий
|
||||
# timeout=3.0; здесь лишь превращаем любую ошибку БД в 503, чтобы
|
||||
# после рестарта PostgreSQL readiness быстро возвращался в 200.
|
||||
raise RuntimeError("db_unavailable: %s" % type(exc).__name__) from exc
|
||||
# Redis (в режиме redis)
|
||||
if _store() == "redis":
|
||||
_redis_client().ping()
|
||||
# S3: HEAD несуществующего ключа. 404/None → S3 доступен.
|
||||
# Короткий таймаут клиента гарантирует быстрый ответ, а ошибка
|
||||
# преобразуется в RuntimeError("s3_unavailable") → 503.
|
||||
try:
|
||||
_s3.head_object("attachments/__health_probe__")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise RuntimeError("s3_unavailable: %s" % type(exc).__name__) from exc
|
||||
return Health(status="ok")
|
||||
@@ -0,0 +1,141 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query
|
||||
from ..models.schemas import CreateMessage, Message, MessagePage, User
|
||||
from ..storage.postgres import storage
|
||||
from .security import current_user
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
|
||||
|
||||
# Контракт: Idempotency-Key ASCII [A-Za-z0-9._:-]{1,128}.
|
||||
_IDEMPOTENCY_ALLOWED = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._:-")
|
||||
|
||||
|
||||
def _require_member(chat_id: UUID, user_id: UUID):
|
||||
chat = storage.get_chat(chat_id)
|
||||
if chat is None or user_id not in chat.member_ids:
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
return chat
|
||||
|
||||
|
||||
def _encode_cursor(chat_id: UUID, message: Message) -> str:
|
||||
data = {"chat_id": str(chat_id), "id": str(message.id),
|
||||
"created_at": message.created_at.isoformat()}
|
||||
return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(chat_id: UUID, cursor: str) -> tuple:
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
|
||||
if data.get("chat_id") != str(chat_id):
|
||||
raise ValueError("cursor_chat_mismatch")
|
||||
return datetime.fromisoformat(data["created_at"]), UUID(data["id"])
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
|
||||
def _canonical_body_hash(payload: CreateMessage) -> str:
|
||||
"""sha256 канонизированного JSON.
|
||||
|
||||
Поле attachment_ids присутствует в модели со значением по умолчанию [],
|
||||
поэтому payload {"text":"x"} и {"text":"x","attachment_ids":[]} дают
|
||||
одинаковый model_dump и одинаковый хэш — как требует контракт.
|
||||
"""
|
||||
data = payload.model_dump(mode="json")
|
||||
canonical = json.dumps(data, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
@router.post(
|
||||
path="/{chat_id}/messages",
|
||||
summary="Отправить сообщение в чат",
|
||||
response_model=Message,
|
||||
status_code=201,
|
||||
)
|
||||
def create_message(
|
||||
chat_id: UUID,
|
||||
payload: CreateMessage,
|
||||
user: User = Depends(current_user),
|
||||
idempotency_key: str = Header(..., alias="Idempotency-Key"),
|
||||
) -> Message:
|
||||
# Строго ASCII, как требует контракт: isalnum() пропускает кириллицу.
|
||||
if not (1 <= len(idempotency_key) <= 128):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
if not all(c in _IDEMPOTENCY_ALLOWED for c in idempotency_key):
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
_require_member(chat_id, user.id)
|
||||
|
||||
text = payload.text or ""
|
||||
attachment_ids = list(payload.attachment_ids)
|
||||
|
||||
# Контракт: text (непустой) ИЛИ attachment_ids (>=1).
|
||||
if text.strip() == "" and not attachment_ids:
|
||||
raise HTTPException(status_code=400, detail="invalid_request")
|
||||
|
||||
# Все вложения должны принадлежать этому чату.
|
||||
if attachment_ids:
|
||||
found = storage.list_attachments_by_ids(attachment_ids, chat_id)
|
||||
if len(found) != len(attachment_ids):
|
||||
raise HTTPException(status_code=404, detail="not_found")
|
||||
for aid in attachment_ids:
|
||||
att, _key = found[aid]
|
||||
if att.status == "failed":
|
||||
raise HTTPException(status_code=409, detail="conflict")
|
||||
|
||||
method = "POST"
|
||||
path = "/api/v1/chats/%s/messages" % chat_id
|
||||
body_hash = _canonical_body_hash(payload)
|
||||
|
||||
# Атомарная операция: INSERT ключа + INSERT сообщения + UPDATE ключа
|
||||
# в ОДНОЙ транзакции. Сбой между ними невозможен: либо всё, либо ничего.
|
||||
state, message, info = storage.add_message_idempotent(
|
||||
chat_id=chat_id,
|
||||
sender_id=user.id,
|
||||
text=text,
|
||||
attachment_ids=attachment_ids,
|
||||
user_id=user.id,
|
||||
method=method,
|
||||
path=path,
|
||||
key=idempotency_key,
|
||||
body_hash=body_hash,
|
||||
)
|
||||
|
||||
if state == "created":
|
||||
return message
|
||||
|
||||
if state == "retry":
|
||||
raise HTTPException(status_code=503, detail="unavailable")
|
||||
|
||||
# state == "replay"
|
||||
if info["body_hash"] != body_hash:
|
||||
raise HTTPException(status_code=409, detail="conflict")
|
||||
if info["response_status"] == 0:
|
||||
# Строка от старой версии кода: не зависаем, отдаём 503.
|
||||
raise HTTPException(status_code=503, detail="unavailable")
|
||||
return Message.model_validate(info["response_body"])
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/{chat_id}/messages",
|
||||
summary="Получить историю сообщений чата",
|
||||
response_model=MessagePage,
|
||||
)
|
||||
def list_messages(
|
||||
chat_id: UUID,
|
||||
user: User = Depends(current_user),
|
||||
limit: int = Query(50, ge=1, le=100),
|
||||
cursor: str | None = None,
|
||||
) -> MessagePage:
|
||||
"""Keyset-пагинация в SQL: (created_at, id) по возрастанию"""
|
||||
_require_member(chat_id, user.id)
|
||||
after = _decode_cursor(chat_id, cursor) if cursor else None
|
||||
rows = storage.list_messages_page(chat_id, limit + 1, after=after)
|
||||
has_more = len(rows) > limit
|
||||
items = rows[:limit]
|
||||
next_cursor = _encode_cursor(chat_id, items[-1]) if has_more and items else None
|
||||
return MessagePage(items=items, next_cursor=next_cursor)
|
||||
@@ -0,0 +1,24 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import HTTPException, Security
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
|
||||
from ..models.schemas import User
|
||||
from ..storage.postgres import storage
|
||||
|
||||
bearer_scheme = HTTPBearer(auto_error=False, scheme_name="bearerAuth")
|
||||
|
||||
|
||||
def current_user(
|
||||
credentials: HTTPAuthorizationCredentials | None = Security(bearer_scheme),
|
||||
) -> User:
|
||||
"""Достаёт Bearer-токен и возвращает владельца сессии.
|
||||
|
||||
Basic на бизнес-endpoint не принимается, X-User-Id игнорируется.
|
||||
"""
|
||||
if credentials is None or credentials.scheme.lower() != "bearer":
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
user = storage.get_session_user(credentials.credentials)
|
||||
if user is None:
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
return user
|
||||
@@ -0,0 +1,31 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from ..models.schemas import CreateUser, User
|
||||
from ..storage.postgres import storage
|
||||
from .security import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["users"])
|
||||
|
||||
|
||||
@router.post(
|
||||
path="/users",
|
||||
summary="Создать пользователя",
|
||||
response_model=User,
|
||||
status_code=201,
|
||||
)
|
||||
def register_user(payload: CreateUser) -> User:
|
||||
"""Публичная регистрация. Пробельные display_name/password отсекаются
|
||||
валидаторами схемы; сюда попадают только валидные значения."""
|
||||
try:
|
||||
return storage.create_user(payload.username, payload.display_name, payload.password)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=409, detail="conflict")
|
||||
|
||||
|
||||
@router.get(
|
||||
path="/users/me",
|
||||
summary="Получить текущего пользователя",
|
||||
response_model=User,
|
||||
)
|
||||
def current_user_me(user: User = Depends(current_user)) -> User:
|
||||
"""Текущий пользователь по Bearer-сессии."""
|
||||
return user
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.openapi.utils import get_openapi
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
import psycopg
|
||||
import uvicorn
|
||||
from opentelemetry import trace as _otel_trace
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
from .api import attachments, auth, chats, health, messages, users
|
||||
from .storage.db import init_pool, close_pool
|
||||
from .telemetry import init_telemetry, INSTANCE_ID, meter, tracer
|
||||
|
||||
|
||||
# Базовый логгер: реальный формат выставляет telemetry.install_log_format()
|
||||
# при вызове init_telemetry(). Здесь — минимальный fallback для импорта.
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
||||
logger = logging.getLogger("app")
|
||||
|
||||
MAX_REQUEST_BODY_BYTES = int(os.environ.get("MAX_REQUEST_BODY_BYTES", "10485760"))
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
init_pool()
|
||||
logger.info(
|
||||
"event=startup app_env=%s session_store=%s instance_id=%s",
|
||||
os.environ.get("APP_ENV", "lab"),
|
||||
os.environ.get("SESSION_STORE", "postgres"),
|
||||
INSTANCE_ID,
|
||||
)
|
||||
yield
|
||||
close_pool()
|
||||
logger.info("event=shutdown_complete instance_id=%s", INSTANCE_ID)
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUCI Messenger — lab6",
|
||||
version="1.6.0",
|
||||
lifespan=lifespan,
|
||||
swagger_ui_parameters={"persistAuthorization": True},
|
||||
)
|
||||
|
||||
# OTel инициализация и FastAPI auto-instrumentation.
|
||||
init_telemetry("messenger-api", "1.6.0")
|
||||
|
||||
|
||||
# --- Метрики HTTP ---
|
||||
_http_counter = meter("messenger").create_counter(
|
||||
"http_requests_total",
|
||||
description="HTTP requests count",
|
||||
unit="1",
|
||||
)
|
||||
# Явные границы бакетов в СЕКУНДАХ, с разрешением в миллисекундном диапазоне
|
||||
# и вокруг порога SLO 2s (upload p95 alert). Дефолтные экспоненциальные
|
||||
# бакеты (0, 5, 10, 25, …) грубы: миллисекундные запросы попадали в один
|
||||
# широкий bucket, а histogram_quantile интерполировал ложные секунды.
|
||||
_HTTP_DURATION_BUCKETS = [
|
||||
0.001, 0.0025, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05,
|
||||
0.075, 0.1, 0.15, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0,
|
||||
]
|
||||
_http_hist = meter("messenger").create_histogram(
|
||||
"http_request_duration_seconds",
|
||||
description="HTTP request duration",
|
||||
unit="s",
|
||||
explicit_bucket_boundaries_advisory=_HTTP_DURATION_BUCKETS,
|
||||
)
|
||||
|
||||
# Фильтр: убираем реальные UUID из маршрута → route template.
|
||||
import re as _re
|
||||
_ROUTE_UUID_RE = _re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
|
||||
|
||||
def _route_template(path: str) -> str:
|
||||
return _ROUTE_UUID_RE.sub("{id}", path)
|
||||
|
||||
|
||||
def custom_openapi():
|
||||
if app.openapi_schema:
|
||||
return app.openapi_schema
|
||||
schema = get_openapi(title=app.title, version=app.version, routes=app.routes)
|
||||
app.openapi_schema = schema
|
||||
return schema
|
||||
|
||||
|
||||
app.openapi = custom_openapi
|
||||
|
||||
# Отдельный tracer для событийного span в middleware. Он создаётся как child
|
||||
# SERVER span, который создаёт OTel FastAPI instrumentation ниже.
|
||||
# ВАЖНО: FastAPIInstrumentor.instrument_app(app) вызывается ПОСЛЕ объявления
|
||||
# middleware, поэтому OTel ASGI middleware оказывается внешним: он извлекает
|
||||
# W3C traceparent из входящих headers, продолжает внешний trace и создаёт
|
||||
# SERVER span. Наш middleware выполняется внутри него, создаёт http.request
|
||||
# как INTERNAL child, и логи/метрики берут корректный trace_id.
|
||||
_http_tracer = tracer("app.http")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def observability_middleware(request: Request, call_next):
|
||||
"""X-Request-Id и X-Instance-Id во всех ответах + структурированный лог + метрики.
|
||||
|
||||
Лог event=http пишется ВНУТРИ активного span (`http.request`), он
|
||||
child SERVER span от OTel auto-instrumentation, поэтому trace_id
|
||||
корректно связывает лог с внешним trace.
|
||||
"""
|
||||
request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
|
||||
request.state.request_id = request_id
|
||||
started = time.monotonic()
|
||||
route = _route_template(request.url.path)
|
||||
|
||||
# lab7: раннее ограничение размера запроса (Content-Length).
|
||||
cl = request.headers.get("content-length")
|
||||
if cl is not None:
|
||||
try:
|
||||
if int(cl) > MAX_REQUEST_BODY_BYTES:
|
||||
response = JSONResponse(
|
||||
status_code=413,
|
||||
content={"error": {"code": "payload_too_large",
|
||||
"message": "payload_too_large",
|
||||
"request_id": request_id}},
|
||||
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
|
||||
)
|
||||
return response
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# http.request — INTERNAL child текущего (SERVER) span, созданного OTel
|
||||
# auto-instrumentation. Parent берётся из активного контекста: входящий
|
||||
# traceparent уже извлечён ASGI middleware.
|
||||
with _http_tracer.start_as_current_span(
|
||||
"http.request",
|
||||
attributes={
|
||||
"http.method": request.method,
|
||||
"http.route": route,
|
||||
},
|
||||
) as span:
|
||||
response = await call_next(request)
|
||||
duration_s = time.monotonic() - started
|
||||
duration_ms = round(duration_s * 1000, 2)
|
||||
status_class = "%dxx" % (response.status_code // 100)
|
||||
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
|
||||
_http_counter.add(
|
||||
1,
|
||||
{"http.method": request.method, "http.route": route,
|
||||
"http.status_class": status_class},
|
||||
)
|
||||
_http_hist.record(
|
||||
duration_s,
|
||||
{"http.method": request.method, "http.route": route,
|
||||
"http.status_class": status_class},
|
||||
)
|
||||
span.set_attribute("http.status_class", status_class)
|
||||
span.set_attribute("http.status_code", response.status_code)
|
||||
|
||||
# Лог ВНУТРИ span → record.trace_id валидный, Loki получает
|
||||
# корреляцию с Tempo.
|
||||
logger.info(
|
||||
"event=http method=%s path=%s status=%s duration_ms=%s "
|
||||
"request_id=%s instance_id=%s",
|
||||
request.method, request.url.path, response.status_code,
|
||||
duration_ms, request_id, INSTANCE_ID,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
# FastAPI auto-instrumentation ДОЛЖНА быть установлена ПОСЛЕ пользовательского
|
||||
# @app.middleware("http"), чтобы оказаться внешним middleware и создать SERVER
|
||||
# span с корректным parent из входного traceparent. Этот вызов также
|
||||
# инструментирует httpx/requests client-вызовы внутри приложения.
|
||||
FastAPIInstrumentor.instrument_app(app)
|
||||
|
||||
|
||||
# --- Обработчики ошибок (без изменений) ---
|
||||
_ERROR_CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
|
||||
def _unauthorized_challenge(path: str) -> str:
|
||||
if path == "/api/v1/auth/sessions":
|
||||
return 'Basic realm="messenger", charset="UTF-8"'
|
||||
return 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def _error_response(status_code: int, message: str, request_id: str,
|
||||
path: str = "", extra_headers: dict | None = None) -> JSONResponse:
|
||||
code = _ERROR_CODES.get(status_code, "internal_error")
|
||||
headers = {}
|
||||
if status_code == 401:
|
||||
headers["WWW-Authenticate"] = _unauthorized_challenge(path)
|
||||
# lab7: пробрасываем заголовки из HTTPException (например, Retry-After у 429).
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
headers["X-Request-Id"] = request_id
|
||||
headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
|
||||
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
|
||||
message = exc.detail if isinstance(exc.detail, str) else "Ошибка"
|
||||
# lab7: сохраняем заголовки, установленные в HTTPException (Retry-After и т.п.).
|
||||
return _error_response(exc.status_code, message, request_id, request.url.path,
|
||||
extra_headers=dict(exc.headers or {}))
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
|
||||
return _error_response(400, "Неверный запрос", request_id, request.url.path)
|
||||
|
||||
|
||||
@app.exception_handler(psycopg.OperationalError)
|
||||
async def db_unavailable_handler(request: Request, exc: psycopg.OperationalError):
|
||||
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
|
||||
logger.error("event=db_unavailable type=%s request_id=%s",
|
||||
type(exc).__name__, request_id)
|
||||
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
|
||||
|
||||
|
||||
@app.exception_handler(RuntimeError)
|
||||
async def runtime_error_handler(request: Request, exc: RuntimeError):
|
||||
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
|
||||
text = str(exc)
|
||||
if "db_unavailable" in text:
|
||||
logger.error("event=db_unavailable request_id=%s", request_id)
|
||||
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
|
||||
if "redis_unavailable" in text:
|
||||
logger.error("event=redis_unavailable request_id=%s", request_id)
|
||||
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
|
||||
if "s3_unavailable" in text:
|
||||
logger.error("event=s3_unavailable request_id=%s", request_id)
|
||||
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
|
||||
logger.error("event=runtime_error type=%s request_id=%s",
|
||||
type(exc).__name__, request_id)
|
||||
return _error_response(500, "Внутренняя ошибка", request_id, request.url.path)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unhandled_exception_handler(request: Request, exc: Exception):
|
||||
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
|
||||
logger.error("event=unhandled_exception type=%s request_id=%s",
|
||||
type(exc).__name__, request_id)
|
||||
return _error_response(500, "Внутренняя ошибка", request_id, request.url.path)
|
||||
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(users.router)
|
||||
app.include_router(chats.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(attachments.router)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
config = uvicorn.Config(
|
||||
"app.main:app",
|
||||
host="0.0.0.0",
|
||||
port=8080,
|
||||
log_level="info",
|
||||
ssl_ca_certs=os.environ.get("TLS_CA_FILE"),
|
||||
ssl_certfile=os.environ.get("TLS_CERT_FILE"),
|
||||
ssl_keyfile=os.environ.get("TLS_KEY_FILE"),
|
||||
ssl_cert_reqs=int(os.environ.get("MTLS_REQUIRE", "0")), # 2 = CERT_REQUIRED
|
||||
)
|
||||
server = uvicorn.Server(config)
|
||||
|
||||
def _handle_sigterm(signum, frame):
|
||||
logger.info("event=sigterm_received signal=%s", signum)
|
||||
server.should_exit = True
|
||||
|
||||
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||
server.run()
|
||||
@@ -0,0 +1,163 @@
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
|
||||
|
||||
|
||||
def _to_z(dt: datetime) -> str:
|
||||
"""RFC 3339 UTC с Z, как требует contracts/README.md."""
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
dt = dt.astimezone(timezone.utc)
|
||||
return dt.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _not_blank(value: str) -> str:
|
||||
"""Контракт: обязательные строки не состоят только из пробельных символов."""
|
||||
if value is None or value.strip() == "":
|
||||
raise ValueError("must not be blank")
|
||||
return value
|
||||
|
||||
|
||||
class Health(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
status: str
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def _display_name_not_blank(cls, v: str) -> str:
|
||||
return _not_blank(v)
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def _password_not_blank(cls, v: str) -> str:
|
||||
return _not_blank(v)
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
@field_serializer("created_at")
|
||||
def _ser_created_at(self, dt: datetime) -> str:
|
||||
return _to_z(dt)
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: List[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def _title_not_blank(cls, v: str) -> str:
|
||||
return _not_blank(v)
|
||||
|
||||
|
||||
class Chat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: List[UUID]
|
||||
created_at: datetime
|
||||
|
||||
@field_serializer("created_at")
|
||||
def _ser_created_at(self, dt: datetime) -> str:
|
||||
return _to_z(dt)
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
"""text может быть пустым, если есть хотя бы одно вложение (lab4)."""
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: List[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def _unique_attachments(cls, v: List[UUID]) -> List[UUID]:
|
||||
if len(set(v)) != len(v):
|
||||
raise ValueError("duplicate attachment_ids")
|
||||
return v
|
||||
|
||||
|
||||
class Message(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: List[UUID] = Field(default_factory=list)
|
||||
|
||||
@field_serializer("created_at")
|
||||
def _ser_created_at(self, dt: datetime) -> str:
|
||||
return _to_z(dt)
|
||||
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
items: List[Chat]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
items: List[Message]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
|
||||
class Session(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
@field_serializer("expires_at")
|
||||
def _ser_expires_at(self, dt: datetime) -> str:
|
||||
return _to_z(dt)
|
||||
|
||||
|
||||
class Variant(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str # "ready"
|
||||
variants: List[Variant] = Field(default_factory=list)
|
||||
error_code: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@field_serializer("created_at")
|
||||
def _ser_created_at(self, dt: datetime) -> str:
|
||||
return _to_z(dt)
|
||||
|
||||
@field_serializer("updated_at")
|
||||
def _ser_updated_at(self, dt: datetime) -> str:
|
||||
return _to_z(dt)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Outbox worker lab5/lab6: publish неопубликованных job в RabbitMQ с трейсингом.
|
||||
|
||||
lab6: payload строки outbox содержит traceparent/tracestate, снятые в момент
|
||||
upload-запроса API. Восстанавливаем контекст, создаём span job.publish как
|
||||
child upload-span и инжектим обновлённый traceparent в headers сообщения.
|
||||
|
||||
Лог event=outbox_published пишется ВНУТРИ span job.publish, чтобы в Loki
|
||||
попадал настоящий trace_id (корреляция лог↔trace).
|
||||
"""
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from opentelemetry import context as otel_context, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
from .storage.db import init_pool, close_pool
|
||||
from .storage.postgres import storage
|
||||
from .storage.rabbit import QUEUE, publish_job
|
||||
from .telemetry import init_telemetry
|
||||
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
||||
log = logging.getLogger("outbox")
|
||||
|
||||
_stop = False
|
||||
_tracer = trace.get_tracer("outbox")
|
||||
_propagator = TraceContextTextMapPropagator()
|
||||
|
||||
init_telemetry("messenger-outbox", "1.6.0")
|
||||
|
||||
|
||||
def _handle_sigterm(signum, frame):
|
||||
global _stop
|
||||
log.info("event=sigterm_received signal=%s", signum)
|
||||
_stop = True
|
||||
|
||||
|
||||
def _extract_parent(payload: dict) -> otel_context.Context:
|
||||
carrier: dict[str, str] = {}
|
||||
if payload.get("traceparent"):
|
||||
carrier["traceparent"] = str(payload["traceparent"])
|
||||
if payload.get("tracestate"):
|
||||
carrier["tracestate"] = str(payload["tracestate"])
|
||||
if not carrier:
|
||||
return otel_context.Context()
|
||||
try:
|
||||
return _propagator.extract(carrier=carrier)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("event=trace_parent_parse_failed type=%s", type(exc).__name__)
|
||||
return otel_context.Context()
|
||||
|
||||
|
||||
def _publish_with_trace(payload: dict, outbox_id: int) -> None:
|
||||
"""Публикует payload; mark_published и лог — внутри span."""
|
||||
parent_ctx = _extract_parent(payload)
|
||||
carrier: dict[str, str] = {}
|
||||
with _tracer.start_as_current_span(
|
||||
"job.publish",
|
||||
context=parent_ctx,
|
||||
kind=SpanKind.PRODUCER,
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE,
|
||||
"job.id": str(payload.get("job_id", "")),
|
||||
"job.type": str(payload.get("type", "unknown")),
|
||||
"outbox.id": int(outbox_id),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
try:
|
||||
_propagator.inject(carrier)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("event=trace_inject_failed type=%s", type(exc).__name__)
|
||||
carrier = {}
|
||||
publish_job(payload, extra_headers=carrier)
|
||||
storage.mark_outbox_published(outbox_id)
|
||||
# Лог внутри span → record.trace_id валидный.
|
||||
log.info("event=outbox_published id=%s job_id=%s",
|
||||
outbox_id, payload.get("job_id"))
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
except Exception as exc:
|
||||
span.record_exception(exc)
|
||||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
raise
|
||||
|
||||
|
||||
def main() -> None:
|
||||
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||
init_pool()
|
||||
log.info("event=outbox_start")
|
||||
|
||||
while not _stop:
|
||||
try:
|
||||
rows = storage.list_unpublished_outbox(limit=50)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=outbox_select_failed type=%s msg=%s",
|
||||
type(exc).__name__, str(exc)[:300])
|
||||
time.sleep(2)
|
||||
continue
|
||||
|
||||
for outbox_id, payload in rows:
|
||||
if _stop:
|
||||
break
|
||||
try:
|
||||
_publish_with_trace(payload, outbox_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"event=outbox_publish_failed id=%s type=%s msg=%s",
|
||||
outbox_id, type(exc).__name__, str(exc)[:300],
|
||||
)
|
||||
time.sleep(2)
|
||||
break
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
close_pool()
|
||||
log.info("event=outbox_stop")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
log.error("event=outbox_fatal\n%s", traceback.format_exc())
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,108 @@
|
||||
"""OpenBao client: AppRole -> scoped token -> KV v2 secret (lab7).
|
||||
|
||||
Root token не используется. AppRole role_id/secret_id приходят из
|
||||
/secrets/*-approle.env, подготовленного openbao-init. Токен живёт
|
||||
коротко (TTL 15m) и приложение не хранит его вечно — при истечении
|
||||
перезапрашивает. Если OpenBao запечатан/недоступен, возвращаем None,
|
||||
и вызывающий код продолжает с уже полученным значением из памяти.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import threading
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
|
||||
_lock = threading.Lock()
|
||||
_cache: dict[str, tuple[str, float]] = {}
|
||||
|
||||
|
||||
def _addr() -> str:
|
||||
return os.environ.get("BAO_ADDR", "https://openbao:8201").rstrip("/")
|
||||
|
||||
|
||||
def _ctx() -> ssl.SSLContext:
|
||||
cafile = os.environ.get("BAO_CACERT", "/certs/ca.crt")
|
||||
return ssl.create_default_context(cafile=cafile if os.path.exists(cafile) else None)
|
||||
|
||||
|
||||
def _read_env(path: str) -> dict:
|
||||
data = {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
data[k.strip()] = v.strip()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _read_approle(role: str) -> tuple[str, str] | None:
|
||||
env = _read_env("/secrets/%s-approle.env" % role)
|
||||
rid = env.get("%s_ROLE_ID" % role.upper())
|
||||
sid = env.get("%s_SECRET_ID" % role.upper())
|
||||
if not rid or not sid:
|
||||
return None
|
||||
return rid, sid
|
||||
|
||||
|
||||
def _post_json(path: str, body: dict, token: str | None = None) -> dict | None:
|
||||
url = _addr() + path
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(url, data=data, method="POST")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
if token:
|
||||
req.add_header("X-Vault-Token", token)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=_ctx(), timeout=5) as r:
|
||||
return json.loads(r.read() or b"{}")
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _get_json(path: str, token: str) -> dict | None:
|
||||
url = _addr() + path
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
req.add_header("X-Vault-Token", token)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=_ctx(), timeout=5) as r:
|
||||
return json.loads(r.read() or b"{}")
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def login_approle(role: str) -> str | None:
|
||||
creds = _read_approle(role)
|
||||
if creds is None:
|
||||
return None
|
||||
rid, sid = creds
|
||||
resp = _post_json("/v1/auth/approle/login", {"role_id": rid, "secret_id": sid})
|
||||
if not resp or "auth" not in resp:
|
||||
return None
|
||||
return resp["auth"].get("client_token")
|
||||
|
||||
|
||||
def get_secret(role: str, key_path: str, field: str) -> str | None:
|
||||
"""role: 'api' | 'worker'. key_path: 'app/api'. Возвращает поле data[field]."""
|
||||
cache_key = "%s:%s:%s" % (role, key_path, field)
|
||||
with _lock:
|
||||
cached = _cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached[0]
|
||||
token = login_approle(role)
|
||||
if token is None:
|
||||
return None
|
||||
resp = _get_json("/v1/secret/data/%s" % key_path, token)
|
||||
if not resp:
|
||||
return None
|
||||
value = (resp.get("data", {}).get("data", {}) or {}).get(field)
|
||||
if value is None:
|
||||
return None
|
||||
with _lock:
|
||||
_cache[cache_key] = (value, 0.0)
|
||||
return value
|
||||
@@ -0,0 +1,225 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
|
||||
_pool: ConnectionPool | None = None
|
||||
|
||||
|
||||
def _dsn() -> str:
|
||||
dsn = os.environ.get("DATABASE_URL")
|
||||
if not dsn:
|
||||
raise RuntimeError("DATABASE_URL is not set")
|
||||
return dsn
|
||||
|
||||
|
||||
def init_pool() -> None:
|
||||
"""Ограниченный пул соединений с таймаутами. Вызывается один раз на старте.
|
||||
|
||||
lab7: API→PostgreSQL идёт по TLS с проверкой сервера (sslmode=verify-full),
|
||||
CA монтируется в /certs/ca.crt (см. compose.yaml). Параметры передаются
|
||||
через kwargs — они применяются независимо от того, что написано в DSN,
|
||||
чтобы .env оставался совместимым с прошлыми лабами.
|
||||
"""
|
||||
global _pool
|
||||
if _pool is not None:
|
||||
return
|
||||
ssl_kwargs = {}
|
||||
ca_file = os.environ.get("TLS_CA_FILE", "/certs/ca.crt")
|
||||
# Если CA нет (неожиданно), не включаем verify-full, чтобы не сломать
|
||||
# локальный запуск без certs; в compose CA всегда смонтирован.
|
||||
if os.path.exists(ca_file):
|
||||
ssl_kwargs["sslmode"] = "verify-full"
|
||||
ssl_kwargs["sslrootcert"] = ca_file
|
||||
_pool = ConnectionPool(
|
||||
conninfo=_dsn(),
|
||||
min_size=2,
|
||||
max_size=10,
|
||||
# Короткий таймаут получения соединения из пула: readiness/бизнес-запрос
|
||||
# при мёртвой БД должен быстро вернуть 503, а не висеть.
|
||||
timeout=3.0,
|
||||
max_idle=300.0,
|
||||
# check: перед выдачей соединения проверяем его живость; мёртвые
|
||||
# соединения после рестарта PostgreSQL отбрасываются, пул сам
|
||||
# открывает новые — readiness восстанавливается без рестарта API.
|
||||
check=ConnectionPool.check_connection,
|
||||
# Раз в 30с фоновый worker пула добивает min_size живыми соединениями,
|
||||
# чтобы после старта БД готовность вернулась в пределах окна теста.
|
||||
reconnect_timeout=30.0,
|
||||
kwargs={"options": "-c statement_timeout=5000", **ssl_kwargs},
|
||||
open=True,
|
||||
)
|
||||
|
||||
|
||||
def close_pool() -> None:
|
||||
global _pool
|
||||
if _pool is not None:
|
||||
_pool.close()
|
||||
_pool = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection() -> Iterator[psycopg.Connection]:
|
||||
"""Соединение из пула; autocommit=False для явных транзакций."""
|
||||
if _pool is None:
|
||||
init_pool()
|
||||
assert _pool is not None
|
||||
with _pool.connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
-- sessions остаются как fallback, если SESSION_STORE=postgres.
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token TEXT PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
-- lab3: общая идемпотентность POST /messages.
|
||||
-- scope: user_id + method + path + key (см. contracts/README.md).
|
||||
-- body_hash — sha256 канонизированного JSON без attachment_ids=[], чтобы
|
||||
-- отличать повтор «того же» body (201, тот же Message) от «другого» (409).
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
body_hash TEXT NOT NULL,
|
||||
response_status INT NOT NULL,
|
||||
response_body JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, method, path, key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_created
|
||||
ON idempotency_keys (created_at);
|
||||
|
||||
-- lab4: вложения
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL, -- queued/processing/ready/failed
|
||||
error_code TEXT,
|
||||
processing_until TIMESTAMPTZ, -- lease для worker (lab5)
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat
|
||||
ON attachments (chat_id);
|
||||
|
||||
-- связь сообщение↔вложение (порядок сохраняем)
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
position INT NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_msg_att_message
|
||||
ON message_attachments (message_id, position);
|
||||
|
||||
-- lab5: варианты обработки (идемпотентность по (attachment_id, pipeline_version, variant))
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version INT NOT NULL,
|
||||
variant TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
width INT NOT NULL,
|
||||
height INT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
-- lab5 level 4: transactional outbox (разрыв COMMIT → publish)
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID NOT NULL UNIQUE,
|
||||
payload JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
published_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished
|
||||
ON outbox (id) WHERE published_at IS NULL;
|
||||
"""
|
||||
|
||||
|
||||
def init_schema() -> None:
|
||||
"""Идемпотентная миграция: безопасно вызывать многократно.
|
||||
|
||||
Вызывается ОДИН РАЗ отдельным one-shot сервисом migrate
|
||||
(см. compose.yaml). Реплики API миграции больше не выполняют:
|
||||
они ждут service_completed_successfully.
|
||||
"""
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(SCHEMA_SQL)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def run_migrations() -> None:
|
||||
"""Точка входа для one-shot migrate-контейнера.
|
||||
|
||||
Поднимает пул, применяет схему, закрывает пул. Завершается с кодом 0
|
||||
при успехе — этого достаточно для service_completed_successfully.
|
||||
"""
|
||||
init_pool()
|
||||
try:
|
||||
init_schema()
|
||||
finally:
|
||||
close_pool()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Запуск: python -m app.storage.db
|
||||
# (в compose.yaml — command: ["python", "-m", "app.storage.db"])
|
||||
import logging
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
)
|
||||
log = logging.getLogger("migrate")
|
||||
log.info("event=migrations_start")
|
||||
run_migrations()
|
||||
log.info("event=migrations_done")
|
||||
@@ -0,0 +1,177 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..models.schemas import Chat, Message, User
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
"""Время сервера в UTC с суффиксом Z на выходе."""
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# --- Password KDF (stdlib scrypt) ---
|
||||
_SCRYPT_N = 2 ** 14
|
||||
_SCRYPT_R = 8
|
||||
_SCRYPT_P = 1
|
||||
_SCRYPT_DKLEN = 32
|
||||
|
||||
|
||||
def _hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
dk = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=_SCRYPT_N,
|
||||
r=_SCRYPT_R,
|
||||
p=_SCRYPT_P,
|
||||
dklen=_SCRYPT_DKLEN,
|
||||
)
|
||||
return "scrypt$%d$%d$%d$%s$%s" % (
|
||||
_SCRYPT_N, _SCRYPT_R, _SCRYPT_P,
|
||||
salt.hex(), dk.hex(),
|
||||
)
|
||||
|
||||
|
||||
def _verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
scheme, n, r, p, salt_hex, dk_hex = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
dk = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=bytes.fromhex(salt_hex),
|
||||
n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(bytes.fromhex(dk_hex)),
|
||||
)
|
||||
return hmac.compare_digest(dk.hex(), dk_hex)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class MemoryStorage:
|
||||
"""Потокобезопасное In-memory хранилище пользователей, чатов, сообщений и сессий. Все данные исчезают при перезапуске процесса"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._users: Dict[UUID, User] = {}
|
||||
self._usernames: Dict[str, UUID] = {}
|
||||
self._passwords: Dict[UUID, str] = {}
|
||||
self._chats: Dict[UUID, Chat] = {}
|
||||
self._messages: Dict[UUID, List[Message]] = {}
|
||||
# token -> (user_id, expires_at)
|
||||
self._sessions: Dict[str, tuple] = {}
|
||||
|
||||
# ---------- Пользователи ----------
|
||||
def create_user(self, username: str, display_name: str, password: str) -> User:
|
||||
with self._lock:
|
||||
if username in self._usernames:
|
||||
raise ValueError("username_taken")
|
||||
user = User(
|
||||
id=uuid4(),
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
created_at=now_utc(),
|
||||
)
|
||||
self._users[user.id] = user
|
||||
self._usernames[username] = user.id
|
||||
self._passwords[user.id] = _hash_password(password)
|
||||
return user
|
||||
|
||||
def get_user(self, user_id: UUID) -> Optional[User]:
|
||||
with self._lock:
|
||||
return self._users.get(user_id)
|
||||
|
||||
def get_user_by_username(self, username: str) -> Optional[User]:
|
||||
with self._lock:
|
||||
uid = self._usernames.get(username)
|
||||
return self._users.get(uid) if uid else None
|
||||
|
||||
def verify_credentials(self, username: str, password: str) -> Optional[User]:
|
||||
with self._lock:
|
||||
uid = self._usernames.get(username)
|
||||
if uid is None:
|
||||
return None
|
||||
stored = self._passwords.get(uid)
|
||||
if not stored or not _verify_password(password, stored):
|
||||
return None
|
||||
return self._users.get(uid)
|
||||
|
||||
# ---------- Сессии ----------
|
||||
def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple:
|
||||
with self._lock:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = now_utc() + timedelta(seconds=ttl_seconds)
|
||||
self._sessions[token] = (user_id, expires_at)
|
||||
return token, expires_at
|
||||
|
||||
def get_session_user(self, token: str) -> Optional[User]:
|
||||
with self._lock:
|
||||
entry = self._sessions.get(token)
|
||||
if entry is None:
|
||||
return None
|
||||
user_id, expires_at = entry
|
||||
if expires_at <= now_utc():
|
||||
self._sessions.pop(token, None)
|
||||
return None
|
||||
return self._users.get(user_id)
|
||||
|
||||
def revoke_session(self, token: str) -> None:
|
||||
with self._lock:
|
||||
self._sessions.pop(token, None)
|
||||
|
||||
# ---------- Чаты ----------
|
||||
def create_chat(self, title: str, member_ids: List[UUID]) -> Chat:
|
||||
with self._lock:
|
||||
for uid in member_ids:
|
||||
if uid not in self._users:
|
||||
raise ValueError("unknown_member")
|
||||
chat = Chat(
|
||||
id=uuid4(),
|
||||
title=title,
|
||||
member_ids=list(member_ids),
|
||||
created_at=now_utc(),
|
||||
)
|
||||
self._chats[chat.id] = chat
|
||||
self._messages[chat.id] = []
|
||||
return chat
|
||||
|
||||
def get_chat(self, chat_id: UUID) -> Optional[Chat]:
|
||||
with self._lock:
|
||||
return self._chats.get(chat_id)
|
||||
|
||||
def list_chats_for_user(self, user_id: UUID) -> List[Chat]:
|
||||
with self._lock:
|
||||
chats = [c for c in self._chats.values() if user_id in c.member_ids]
|
||||
chats.sort(key=lambda c: (c.created_at, str(c.id)))
|
||||
return chats
|
||||
|
||||
# ---------- Сообщения ----------
|
||||
def add_message(self, chat_id: UUID, sender_id: UUID, text: str) -> Message:
|
||||
"""Улучшение Трека 1: список сообщений хранится уже отсортированным.
|
||||
created_at монотонен в пределах одного процесса, поэтому append
|
||||
сохраняет порядок (created_at, id) по возрастанию — сортировка на
|
||||
чтении не нужна. При конкурентных вставках под RLock порядок
|
||||
сохраняется, но монотонность часов не гарантируется контрактом."""
|
||||
with self._lock:
|
||||
message = Message(
|
||||
id=uuid4(),
|
||||
chat_id=chat_id,
|
||||
sender_id=sender_id,
|
||||
text=text,
|
||||
created_at=now_utc(),
|
||||
)
|
||||
self._messages.setdefault(chat_id, []).append(message)
|
||||
return message
|
||||
|
||||
def list_messages(self, chat_id: UUID) -> List[Message]:
|
||||
with self._lock:
|
||||
# Без пересортировки: список уже в порядке (created_at, id).
|
||||
return list(self._messages.get(chat_id, []))
|
||||
|
||||
|
||||
storage = MemoryStorage() # единственный экземпляр хранилища на процесс
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Argon2id password hashing (lab7).
|
||||
|
||||
Параметры по умолчанию из .env.example: memory=19 MiB, t=2, p=1.
|
||||
Хранимая строка — стандартный encoded-hash Argon2 (`$argon2id$v=19$...`),
|
||||
он уже включает соль и параметры. Rehash старого scrypt-хэша при успешном
|
||||
входе выполняется вызывающим кодом.
|
||||
"""
|
||||
import os
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
# Параметры можно переопределить через .env; значения не ниже методички.
|
||||
_MEMORY_KIB = int(os.environ.get("ARGON2_MEMORY_KIB", "19456"))
|
||||
_ITERATIONS = int(os.environ.get("ARGON2_ITERATIONS", "2"))
|
||||
_PARALLELISM = int(os.environ.get("ARGON2_PARALLELISM", "1"))
|
||||
|
||||
_hasher = PasswordHasher(
|
||||
time_cost=_ITERATIONS,
|
||||
memory_cost=_MEMORY_KIB,
|
||||
parallelism=_PARALLELISM,
|
||||
hash_len=32,
|
||||
salt_len=16,
|
||||
)
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Возвращает encoded Argon2id hash (salt+params включены)."""
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
"""True только для корректного пароля к Argon2id-хэшу."""
|
||||
if not stored or not stored.startswith("$argon2id$"):
|
||||
return False
|
||||
try:
|
||||
_hasher.verify(stored, password)
|
||||
return True
|
||||
except (VerifyMismatchError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def needs_rehash(stored: str) -> bool:
|
||||
"""True, если hash устарел по параметрам (Argon2) — обновить при входе."""
|
||||
if not stored or not stored.startswith("$argon2id$"):
|
||||
return False
|
||||
try:
|
||||
return _hasher.check_needs_rehash(stored)
|
||||
except InvalidHashError:
|
||||
return False
|
||||
@@ -0,0 +1,622 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import json as _json
|
||||
import secrets
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..models.schemas import Attachment, Chat, Message, User, Variant
|
||||
from .db import connection
|
||||
from .sessions import (
|
||||
create_session as _sess_create,
|
||||
get_session_user as _sess_get,
|
||||
revoke_session as _sess_revoke,
|
||||
)
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# --- Password KDF (lab7: Argon2id, с rehash со старого scrypt) ---
|
||||
from .passwords import hash_password as _argon_hash, verify_password as _argon_verify, needs_rehash as _argon_needs_rehash
|
||||
|
||||
|
||||
def _hash_password(password: str) -> str:
|
||||
"""Новый пароль хранится только как Argon2id encoded-hash."""
|
||||
return _argon_hash(password)
|
||||
|
||||
|
||||
def _verify_password(password: str, stored: str) -> bool:
|
||||
"""Проверяет Argon2id; старый scrypt-хэш тоже принимается (для rehash)."""
|
||||
if not stored:
|
||||
return False
|
||||
if stored.startswith("$argon2id$"):
|
||||
return _argon_verify(password, stored)
|
||||
# Совместимость с ранее сохранёнными scrypt-хэшами (одноразово).
|
||||
try:
|
||||
scheme, n, r, p, salt_hex, dk_hex = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
dk = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=bytes.fromhex(salt_hex),
|
||||
n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(bytes.fromhex(dk_hex)),
|
||||
)
|
||||
return hmac.compare_digest(dk.hex(), dk_hex)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _row_to_user(row) -> User:
|
||||
return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3])
|
||||
|
||||
|
||||
def _row_to_chat(row, member_ids: List[UUID]) -> Chat:
|
||||
return Chat(id=row[0], title=row[1], member_ids=member_ids, created_at=row[2])
|
||||
|
||||
|
||||
def _row_to_message(row, attachment_ids: List[UUID]) -> Message:
|
||||
return Message(
|
||||
id=row[0], chat_id=row[1], sender_id=row[2],
|
||||
text=row[3], created_at=row[4],
|
||||
attachment_ids=attachment_ids,
|
||||
)
|
||||
|
||||
|
||||
def _row_to_attachment(row, variants: Optional[list] = None) -> Attachment:
|
||||
# row: id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
# status, error_code, created_at, updated_at
|
||||
return Attachment(
|
||||
id=row[0], chat_id=row[1], owner_id=row[2],
|
||||
filename=row[3], content_type=row[4], size_bytes=row[5],
|
||||
sha256=row[6], status=row[7], variants=variants or [],
|
||||
error_code=row[8],
|
||||
created_at=row[9], updated_at=row[10],
|
||||
)
|
||||
|
||||
|
||||
class PostgresStorage:
|
||||
"""Потокобезопасная обёртка; параллелизм обеспечивает PostgreSQL."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
|
||||
# ---------- Пользователи ----------
|
||||
def create_user(self, username: str, display_name: str, password: str) -> User:
|
||||
with self._lock, connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM users WHERE username = %s", (username,))
|
||||
if cur.fetchone() is not None:
|
||||
raise ValueError("username_taken")
|
||||
user = User(
|
||||
id=uuid4(), username=username,
|
||||
display_name=display_name, created_at=now_utc(),
|
||||
)
|
||||
try:
|
||||
cur.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s)",
|
||||
(user.id, user.username, user.display_name,
|
||||
_hash_password(password), user.created_at),
|
||||
)
|
||||
except Exception as exc:
|
||||
if exc.__class__.__name__ == "UniqueViolation":
|
||||
raise ValueError("username_taken")
|
||||
raise
|
||||
conn.commit()
|
||||
return user
|
||||
|
||||
def get_user(self, user_id: UUID) -> Optional[User]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return _row_to_user(row) if row else None
|
||||
|
||||
def get_user_by_username(self, username: str) -> Optional[User]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE username = %s",
|
||||
(username,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return _row_to_user(row) if row else None
|
||||
|
||||
def verify_credentials(self, username: str, password: str) -> Optional[User]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash"
|
||||
" FROM users WHERE username = %s",
|
||||
(username,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
stored = row[4]
|
||||
if not _verify_password(password, stored):
|
||||
return None
|
||||
# lab7: rehash старого/устаревшего хэша при успешном входе.
|
||||
if not stored.startswith("$argon2id$") or _argon_needs_rehash(stored):
|
||||
try:
|
||||
cur.execute(
|
||||
"UPDATE users SET password_hash = %s WHERE id = %s",
|
||||
(_argon_hash(password), row[0]),
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
# Rehash не должен ломать успешный вход.
|
||||
conn.rollback()
|
||||
return _row_to_user(row[:4])
|
||||
|
||||
# ---------- Сессии ----------
|
||||
def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple:
|
||||
return _sess_create(user_id, ttl_seconds)
|
||||
|
||||
def get_session_user(self, token: str) -> Optional[User]:
|
||||
return _sess_get(token)
|
||||
|
||||
def revoke_session(self, token: str) -> None:
|
||||
_sess_revoke(token)
|
||||
|
||||
# ---------- Чаты ----------
|
||||
def create_chat(self, title: str, member_ids: List[UUID]) -> Chat:
|
||||
with self._lock, connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
# проверяем всех участников
|
||||
cur.execute(
|
||||
"SELECT id FROM users WHERE id = ANY(%s)",
|
||||
(list(member_ids),),
|
||||
)
|
||||
found = {r[0] for r in cur.fetchall()}
|
||||
if len(found) != len(set(member_ids)):
|
||||
raise ValueError("unknown_member")
|
||||
chat = Chat(
|
||||
id=uuid4(), title=title,
|
||||
member_ids=list(member_ids), created_at=now_utc(),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
|
||||
(chat.id, chat.title, chat.created_at),
|
||||
)
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
[(chat.id, uid) for uid in member_ids],
|
||||
)
|
||||
conn.commit()
|
||||
return chat
|
||||
|
||||
def get_chat(self, chat_id: UUID) -> Optional[Chat]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, title, created_at FROM chats WHERE id = %s",
|
||||
(chat_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
cur.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id,),
|
||||
)
|
||||
members = [r[0] for r in cur.fetchall()]
|
||||
return _row_to_chat(row, members)
|
||||
|
||||
def list_chats_for_user(self, user_id: UUID) -> List[Chat]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c"
|
||||
" JOIN chat_members m ON m.chat_id = c.id"
|
||||
" WHERE m.user_id = %s ORDER BY c.created_at, c.id",
|
||||
(user_id,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
chats = []
|
||||
for row in rows:
|
||||
cur.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s", (row[0],),
|
||||
)
|
||||
members = [r[0] for r in cur.fetchall()]
|
||||
chats.append(_row_to_chat(row, members))
|
||||
return chats
|
||||
|
||||
# ---------- Сообщения ----------
|
||||
def _load_message_attachments(self, cur, message_ids: List[UUID]) -> dict[UUID, List[UUID]]:
|
||||
if not message_ids:
|
||||
return {}
|
||||
cur.execute(
|
||||
"SELECT message_id, attachment_id, position FROM message_attachments"
|
||||
" WHERE message_id = ANY(%s) ORDER BY message_id, position",
|
||||
(list(message_ids),),
|
||||
)
|
||||
result: dict[UUID, List[UUID]] = {}
|
||||
for mid, aid, _pos in cur.fetchall():
|
||||
result.setdefault(mid, []).append(aid)
|
||||
return result
|
||||
|
||||
def list_messages_page(
|
||||
self, chat_id: UUID, limit: int, after: Optional[tuple] = None,
|
||||
) -> List[Message]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
if after is None:
|
||||
cur.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages"
|
||||
" WHERE chat_id = %s ORDER BY created_at, id LIMIT %s",
|
||||
(chat_id, limit),
|
||||
)
|
||||
else:
|
||||
after_ts, after_id = after
|
||||
cur.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages"
|
||||
" WHERE chat_id = %s AND (created_at, id) > (%s, %s)"
|
||||
" ORDER BY created_at, id LIMIT %s",
|
||||
(chat_id, after_ts, after_id, limit),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
ids = [r[0] for r in rows]
|
||||
att_map = self._load_message_attachments(cur, ids)
|
||||
return [_row_to_message(r, att_map.get(r[0], [])) for r in rows]
|
||||
|
||||
# ---------- Идемпотентность (lab3) ----------
|
||||
def add_message_idempotent(
|
||||
self,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
attachment_ids: List[UUID],
|
||||
user_id: UUID,
|
||||
method: str,
|
||||
path: str,
|
||||
key: str,
|
||||
body_hash: str,
|
||||
) -> tuple[str, Optional[Message], Optional[dict]]:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO idempotency_keys"
|
||||
" (user_id, method, path, key, body_hash, response_status, response_body)"
|
||||
" VALUES (%s, %s, %s, %s, %s, 0, '{}'::jsonb)"
|
||||
" ON CONFLICT (user_id, method, path, key) DO NOTHING"
|
||||
" RETURNING key",
|
||||
(user_id, method, path, key, body_hash),
|
||||
)
|
||||
won = cur.fetchone() is not None
|
||||
|
||||
if won:
|
||||
message = Message(
|
||||
id=uuid4(), chat_id=chat_id, sender_id=sender_id,
|
||||
text=text, created_at=now_utc(),
|
||||
attachment_ids=list(attachment_ids),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s)",
|
||||
(message.id, message.chat_id, message.sender_id,
|
||||
message.text, message.created_at),
|
||||
)
|
||||
if attachment_ids:
|
||||
cur.executemany(
|
||||
"INSERT INTO message_attachments"
|
||||
" (message_id, attachment_id, position)"
|
||||
" VALUES (%s, %s, %s)",
|
||||
[(message.id, aid, idx)
|
||||
for idx, aid in enumerate(attachment_ids)],
|
||||
)
|
||||
body_json = _json.loads(message.model_dump_json())
|
||||
cur.execute(
|
||||
"UPDATE idempotency_keys"
|
||||
" SET response_status = %s, response_body = %s"
|
||||
" WHERE user_id = %s AND method = %s AND path = %s AND key = %s",
|
||||
(201, _json.dumps(body_json),
|
||||
user_id, method, path, key),
|
||||
)
|
||||
conn.commit()
|
||||
return ("created", message, None)
|
||||
|
||||
cur.execute(
|
||||
"SELECT body_hash, response_status, response_body"
|
||||
" FROM idempotency_keys"
|
||||
" WHERE user_id = %s AND method = %s AND path = %s AND key = %s",
|
||||
(user_id, method, path, key),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.commit()
|
||||
|
||||
if row is None:
|
||||
return ("retry", None, None)
|
||||
return (
|
||||
"replay",
|
||||
None,
|
||||
{
|
||||
"body_hash": row[0],
|
||||
"response_status": row[1],
|
||||
"response_body": row[2],
|
||||
},
|
||||
)
|
||||
|
||||
# ---------- Вложения ----------
|
||||
def create_attachment(self, attachment: Attachment, object_key: str) -> Attachment:
|
||||
"""lab4 fallback (синхронный ready). Не используется в lab5."""
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO attachments"
|
||||
" (id, chat_id, owner_id, filename, content_type, size_bytes,"
|
||||
" sha256, object_key, status, error_code, created_at, updated_at)"
|
||||
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||
(
|
||||
attachment.id, attachment.chat_id, attachment.owner_id,
|
||||
attachment.filename, attachment.content_type,
|
||||
attachment.size_bytes, attachment.sha256, object_key,
|
||||
attachment.status, attachment.error_code,
|
||||
attachment.created_at, attachment.updated_at,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return attachment
|
||||
|
||||
def create_attachment_queued(
|
||||
self,
|
||||
attachment: Attachment,
|
||||
staging_key: str,
|
||||
outbox_payload: dict,
|
||||
) -> Attachment:
|
||||
"""lab5: attachment в queued + строка outbox в ОДНОЙ транзакции."""
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO attachments"
|
||||
" (id, chat_id, owner_id, filename, content_type, size_bytes,"
|
||||
" sha256, object_key, status, error_code, processing_until,"
|
||||
" created_at, updated_at)"
|
||||
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||
(
|
||||
attachment.id, attachment.chat_id, attachment.owner_id,
|
||||
attachment.filename, attachment.content_type,
|
||||
attachment.size_bytes, attachment.sha256, staging_key,
|
||||
"queued", None, None,
|
||||
attachment.created_at, attachment.updated_at,
|
||||
),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO outbox (job_id, payload) VALUES (%s, %s)",
|
||||
(outbox_payload["job_id"], _json.dumps(outbox_payload)),
|
||||
)
|
||||
conn.commit()
|
||||
return attachment
|
||||
|
||||
def claim_processing(self, attachment_id: UUID, lease_seconds: int = 60) -> bool:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
now = now_utc()
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='processing',"
|
||||
" processing_until=%s, updated_at=%s"
|
||||
" WHERE id=%s AND status IN ('queued','processing')"
|
||||
" RETURNING id",
|
||||
(now + timedelta(seconds=lease_seconds), now, attachment_id),
|
||||
)
|
||||
won = cur.fetchone() is not None
|
||||
conn.commit()
|
||||
return won
|
||||
|
||||
def recover_stuck_processing(self) -> int:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
now = now_utc()
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='queued', processing_until=NULL,"
|
||||
" updated_at=%s"
|
||||
" WHERE status='processing' AND processing_until IS NOT NULL"
|
||||
" AND processing_until < %s",
|
||||
(now, now),
|
||||
)
|
||||
count = cur.rowcount
|
||||
conn.commit()
|
||||
return count
|
||||
|
||||
def replay_failed_to_queued(self, attachment_id: UUID) -> str:
|
||||
"""Операторский replay: failed → queued с новой записью в outbox.
|
||||
|
||||
Возвращает:
|
||||
- "requeued" — статус был failed, переведён в queued и запланирован job;
|
||||
- "already" — запись уже queued/processing/ready (защита готовых), ничего не меняем;
|
||||
- "missing" — такого вложения нет.
|
||||
|
||||
Ключ эффекта (attachment_id, pipeline_version, variant) и ON CONFLICT
|
||||
в finalize_ready гарантируют, что повтор не создаст дублей вариантов.
|
||||
Новый job_id и строка outbox публикуются отдельным процессом outbox,
|
||||
поэтому разрыв между COMMIT и publish закрыт тем же механизмом, что и при upload.
|
||||
"""
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT status FROM attachments WHERE id = %s FOR UPDATE",
|
||||
(attachment_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
conn.rollback()
|
||||
return "missing"
|
||||
status = row[0]
|
||||
if status != "failed":
|
||||
conn.rollback()
|
||||
return "already"
|
||||
|
||||
now = now_utc()
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='queued', error_code=NULL,"
|
||||
" processing_until=NULL, updated_at=%s WHERE id=%s",
|
||||
(now, attachment_id),
|
||||
)
|
||||
|
||||
job_id = uuid4()
|
||||
payload = {
|
||||
"job_id": str(job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": 1,
|
||||
"created_at": now.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
cur.execute(
|
||||
"INSERT INTO outbox (job_id, payload) VALUES (%s, %s)",
|
||||
(job_id, _json.dumps(payload)),
|
||||
)
|
||||
conn.commit()
|
||||
return "requeued"
|
||||
|
||||
def finalize_ready(
|
||||
self,
|
||||
attachment_id: UUID,
|
||||
pipeline_version: int,
|
||||
variants: list[dict],
|
||||
) -> None:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
original = next(v for v in variants if v["name"] == "original")
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='ready', error_code=NULL,"
|
||||
" object_key=%s, processing_until=NULL, updated_at=%s"
|
||||
" WHERE id=%s",
|
||||
(original["object_key"], now_utc(), attachment_id),
|
||||
)
|
||||
for v in variants:
|
||||
if v["name"] == "original":
|
||||
continue
|
||||
cur.execute(
|
||||
"INSERT INTO attachment_variants"
|
||||
" (attachment_id, pipeline_version, variant, object_key,"
|
||||
" content_type, size_bytes, width, height, sha256)"
|
||||
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
" ON CONFLICT (attachment_id, pipeline_version, variant)"
|
||||
" DO NOTHING",
|
||||
(
|
||||
attachment_id, pipeline_version, v["name"],
|
||||
v["object_key"], v["content_type"],
|
||||
v["size_bytes"], v["width"], v["height"], v["sha256"],
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def finalize_failed(self, attachment_id: UUID, error_code: str) -> None:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='failed', error_code=%s,"
|
||||
" processing_until=NULL, updated_at=%s WHERE id=%s",
|
||||
(error_code, now_utc(), attachment_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get_attachment(self, attachment_id: UUID) -> Optional[Attachment]:
|
||||
pair = self.get_attachment_with_key(attachment_id)
|
||||
return pair[0] if pair else None
|
||||
|
||||
def get_attachment_with_key(
|
||||
self, attachment_id: UUID
|
||||
) -> Optional[tuple[Attachment, str]]:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes,"
|
||||
" sha256, status, error_code, created_at, updated_at, object_key"
|
||||
" FROM attachments WHERE id = %s",
|
||||
(attachment_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
att = _row_to_attachment(row[:11], self.get_attachment_variants(attachment_id))
|
||||
return att, row[11]
|
||||
|
||||
def list_attachments_by_ids(
|
||||
self, ids: List[UUID], chat_id: UUID
|
||||
) -> dict[UUID, tuple[Attachment, str]]:
|
||||
if not ids:
|
||||
return {}
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes,"
|
||||
" sha256, status, error_code, created_at, updated_at, object_key"
|
||||
" FROM attachments WHERE chat_id = %s AND id = ANY(%s)",
|
||||
(chat_id, list(ids)),
|
||||
)
|
||||
result: dict[UUID, tuple[Attachment, str]] = {}
|
||||
for row in cur.fetchall():
|
||||
att = _row_to_attachment(row[:11])
|
||||
result[row[0]] = (att, row[11])
|
||||
return result
|
||||
|
||||
def delete_attachment(self, attachment_id: UUID) -> None:
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM attachments WHERE id = %s", (attachment_id,))
|
||||
conn.commit()
|
||||
|
||||
def get_attachment_variants(self, attachment_id: UUID) -> list[dict]:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT variant, content_type, size_bytes, width, height, sha256"
|
||||
" FROM attachment_variants WHERE attachment_id=%s"
|
||||
" ORDER BY variant",
|
||||
(attachment_id,),
|
||||
)
|
||||
return [
|
||||
{
|
||||
"name": r[0], "content_type": r[1], "size_bytes": r[2],
|
||||
"width": r[3], "height": r[4], "sha256": r[5],
|
||||
}
|
||||
for r in cur.fetchall()
|
||||
]
|
||||
|
||||
def get_staging_key(self, attachment_id: UUID) -> Optional[str]:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT object_key FROM attachments WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
# ---------- Outbox (lab5 level 4) ----------
|
||||
def list_unpublished_outbox(self, limit: int = 100) -> list[tuple[int, dict]]:
|
||||
"""Возвращает [(outbox_id, payload_dict), ...].
|
||||
|
||||
Всегда dict; если payload в БД пришёл строкой (например, из старой
|
||||
версии), распаковываем через json.loads.
|
||||
"""
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, payload FROM outbox WHERE published_at IS NULL"
|
||||
" ORDER BY id LIMIT %s",
|
||||
(limit,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
result: list[tuple[int, dict]] = []
|
||||
for r in rows:
|
||||
raw = r[1]
|
||||
if isinstance(raw, dict):
|
||||
payload = raw
|
||||
else:
|
||||
payload = _json.loads(raw)
|
||||
if not isinstance(payload, dict):
|
||||
# защита от мусора — пропустить
|
||||
continue
|
||||
result.append((int(r[0]), payload))
|
||||
return result
|
||||
|
||||
def mark_outbox_published(self, outbox_id: int) -> None:
|
||||
with connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE outbox SET published_at=now() WHERE id=%s",
|
||||
(outbox_id,),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
storage = PostgresStorage() # единственный экземпляр на процесс
|
||||
@@ -0,0 +1,155 @@
|
||||
"""RabbitMQ: topology, publisher with confirms, consumer с manual ack.
|
||||
|
||||
- Основная очередь: durable quorum queue.
|
||||
- Retry: TTL-очереди с dead-letter обратно в основную; x-retry-count.
|
||||
- DLQ: финальные ошибки и исчерпанные retry.
|
||||
- Publisher confirms: publish() ждёт подтверждения брокера.
|
||||
- lab6: extra_headers для traceparent/tracestate.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
import pika
|
||||
|
||||
log = logging.getLogger("rabbit")
|
||||
|
||||
EXCHANGE = "messenger.attachments"
|
||||
QUEUE = "attachment.process.v1"
|
||||
RETRY_EXCHANGE = "messenger.attachments.retry"
|
||||
DLX = "messenger.attachments.dlx"
|
||||
DLQ = "attachment.process.v1.dlq"
|
||||
|
||||
RETRY_DELAYS = [5, 15, 60]
|
||||
MAX_ATTEMPTS = len(RETRY_DELAYS) + 1
|
||||
|
||||
_topology_ready = False
|
||||
_topology_lock = threading.Lock()
|
||||
|
||||
|
||||
def _url() -> str:
|
||||
return os.environ.get("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/%2F")
|
||||
|
||||
|
||||
def connection_params() -> pika.ConnectionParameters:
|
||||
return pika.URLParameters(_url())
|
||||
|
||||
|
||||
def connect() -> pika.BlockingConnection:
|
||||
return pika.BlockingConnection(connection_params())
|
||||
|
||||
|
||||
def setup_topology(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
|
||||
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
|
||||
channel.exchange_declare(RETRY_EXCHANGE, exchange_type="direct", durable=True)
|
||||
channel.exchange_declare(DLX, exchange_type="direct", durable=True)
|
||||
|
||||
channel.queue_declare(DLQ, durable=True, arguments={"x-queue-type": "quorum"})
|
||||
channel.queue_bind(DLQ, DLX, routing_key=DLQ)
|
||||
|
||||
channel.queue_declare(
|
||||
QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-queue-type": "quorum",
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-dead-letter-routing-key": DLQ,
|
||||
},
|
||||
)
|
||||
channel.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE)
|
||||
|
||||
for delay in RETRY_DELAYS:
|
||||
qname = f"attachment.retry.{delay}s"
|
||||
channel.queue_declare(
|
||||
qname,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-message-ttl": delay * 1000,
|
||||
"x-dead-letter-exchange": EXCHANGE,
|
||||
"x-dead-letter-routing-key": QUEUE,
|
||||
},
|
||||
)
|
||||
channel.queue_bind(qname, RETRY_EXCHANGE, routing_key=qname)
|
||||
|
||||
|
||||
def _ensure_topology(ch: pika.adapters.blocking_connection.BlockingChannel) -> None:
|
||||
global _topology_ready
|
||||
if _topology_ready:
|
||||
return
|
||||
with _topology_lock:
|
||||
if _topology_ready:
|
||||
return
|
||||
setup_topology(ch)
|
||||
_topology_ready = True
|
||||
|
||||
|
||||
def publish_job(payload: dict, extra_headers: Optional[dict] = None) -> None:
|
||||
"""Публикация job с publisher confirm. extra_headers — traceparent."""
|
||||
conn = connect()
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.confirm_delivery()
|
||||
_ensure_topology(ch)
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
headers = dict(extra_headers or {})
|
||||
ch.basic_publish(
|
||||
exchange=EXCHANGE,
|
||||
routing_key=QUEUE,
|
||||
body=body.encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
headers=headers or None,
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def publish_retry(payload: dict, retry_count: int) -> None:
|
||||
idx = min(retry_count - 1, len(RETRY_DELAYS) - 1)
|
||||
delay = RETRY_DELAYS[idx]
|
||||
qname = f"attachment.retry.{delay}s"
|
||||
conn = connect()
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.confirm_delivery()
|
||||
_ensure_topology(ch)
|
||||
headers = dict(payload.get("_headers") or {})
|
||||
headers["x-retry-count"] = retry_count
|
||||
clean = {k: v for k, v in payload.items() if k != "_headers"}
|
||||
body = json.dumps(clean, ensure_ascii=False, separators=(",", ":"))
|
||||
ch.basic_publish(
|
||||
exchange=RETRY_EXCHANGE,
|
||||
routing_key=qname,
|
||||
body=body.encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
headers=headers,
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def publish_dlq(payload: dict) -> None:
|
||||
conn = connect()
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.confirm_delivery()
|
||||
_ensure_topology(ch)
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
ch.basic_publish(
|
||||
exchange=DLX,
|
||||
routing_key=DLQ,
|
||||
body=body.encode("utf-8"),
|
||||
properties=pika.BasicProperties(delivery_mode=2, content_type="application/json"),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Операторский replay failed-задач: failed → queued + запись в outbox.
|
||||
|
||||
Использование:
|
||||
python -m app.storage.replay_failed <attachment_id> [<attachment_id> ...]
|
||||
python -m app.storage.replay_failed --all-failed
|
||||
|
||||
Защита:
|
||||
- переводит ТОЛЬКО failed; queued/processing/ready не трогает (возвращает "already");
|
||||
- новая строка outbox с новым job_id — публикация через существующий outbox-процесс;
|
||||
- идемпотентность эффекта по (attachment_id, pipeline_version, variant)
|
||||
не даёт создать дубли вариантов, если повторная обработка уже была выполнена.
|
||||
Логи пишутся только с ID и статусом, без секретов и содержимого файлов.
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from .db import init_pool, close_pool
|
||||
from .postgres import storage
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
log = logging.getLogger("replay_failed")
|
||||
|
||||
|
||||
def _parse_args(argv):
|
||||
if not argv:
|
||||
print(
|
||||
"usage: python -m app.storage.replay_failed <attachment_id> [...] | --all-failed",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return None
|
||||
if argv == ["--all-failed"]:
|
||||
return "all"
|
||||
ids = []
|
||||
for raw in argv:
|
||||
try:
|
||||
ids.append(uuid.UUID(raw))
|
||||
except ValueError:
|
||||
print("invalid attachment_id: %r" % raw, file=sys.stderr)
|
||||
return None
|
||||
return ids
|
||||
|
||||
|
||||
def _list_failed_ids() -> list:
|
||||
from .db import connection
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM attachments WHERE status='failed' ORDER BY updated_at")
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
|
||||
|
||||
def main(argv) -> int:
|
||||
parsed = _parse_args(argv)
|
||||
if parsed is None:
|
||||
return 2
|
||||
init_pool()
|
||||
try:
|
||||
if parsed == "all":
|
||||
ids = _list_failed_ids()
|
||||
else:
|
||||
ids = parsed
|
||||
requeued = already = missing = 0
|
||||
for aid in ids:
|
||||
result = storage.replay_failed_to_queued(aid)
|
||||
if result == "requeued":
|
||||
requeued += 1
|
||||
log.info("event=replay_requeued attachment_id=%s", aid)
|
||||
elif result == "already":
|
||||
already += 1
|
||||
log.info("event=replay_skipped_not_failed attachment_id=%s", aid)
|
||||
else:
|
||||
missing += 1
|
||||
log.info("event=replay_missing attachment_id=%s", aid)
|
||||
log.info(
|
||||
"event=replay_done requested=%d requeued=%d skipped=%d missing=%d",
|
||||
len(ids), requeued, already, missing,
|
||||
)
|
||||
return 0 if missing == 0 else 1
|
||||
finally:
|
||||
close_pool()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,98 @@
|
||||
"""S3-клиент (MinIO) для lab4/lab5.
|
||||
|
||||
Object key генерирует сервер:
|
||||
staging: staging/<chat_id>/<attachment_id>/original
|
||||
final: attachments/<chat_id>/<attachment_id>/original
|
||||
variant: attachments/<chat_id>/<attachment_id>/<variant>
|
||||
filename используется только для отображения, не в ключе.
|
||||
"""
|
||||
import os
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
def _endpoint() -> str:
|
||||
return os.environ["S3_ENDPOINT"]
|
||||
|
||||
|
||||
def _bucket() -> str:
|
||||
return os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
|
||||
|
||||
def client():
|
||||
"""Клиент с прикладными credentials (не admin).
|
||||
|
||||
Явные короткие таймауты: connect/read — чтобы при недоступности S3
|
||||
запросы падали быстро и LB отдавал 503, а не 504 по своему timeout.
|
||||
"""
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=_endpoint(),
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ["S3_APP_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["S3_APP_SECRET_KEY"],
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=3,
|
||||
retries={"max_attempts": 0},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def staging_key(chat_id: UUID, attachment_id: UUID) -> str:
|
||||
"""Staging — приватный prefix, worker читает отсюда."""
|
||||
return "staging/%s/%s/original" % (chat_id, attachment_id)
|
||||
|
||||
|
||||
def original_key(chat_id: UUID, attachment_id: UUID) -> str:
|
||||
"""Финальный ключ оригинала. Не зависит от filename."""
|
||||
return "attachments/%s/%s/original" % (chat_id, attachment_id)
|
||||
|
||||
|
||||
def variant_key(chat_id: UUID, attachment_id: UUID, variant: str) -> str:
|
||||
return "attachments/%s/%s/%s" % (chat_id, attachment_id, variant)
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str) -> None:
|
||||
client().put_object(
|
||||
Bucket=_bucket(),
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
ContentLength=len(data),
|
||||
)
|
||||
|
||||
|
||||
def get_object(key: str):
|
||||
"""Возвращает (bytes, content_type). Бросает ClientError при отсутствии."""
|
||||
resp = client().get_object(Bucket=_bucket(), Key=key)
|
||||
body = resp["Body"].read()
|
||||
ctype = resp.get("ContentType") or "application/octet-stream"
|
||||
return body, ctype
|
||||
|
||||
|
||||
def delete_object(key: str) -> None:
|
||||
client().delete_object(Bucket=_bucket(), Key=key)
|
||||
|
||||
|
||||
def head_object(key: str) -> Optional[dict]:
|
||||
try:
|
||||
return client().head_object(Bucket=_bucket(), Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def list_objects(prefix: str = "attachments/"):
|
||||
"""Итератор по объектам bucket. Используется cleanup-сервисом."""
|
||||
paginator = client().get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=_bucket(), Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
yield obj
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Фоновая сверка S3 и PostgreSQL (уровень 4 lab4/lab5).
|
||||
|
||||
- Orphan: объект в S3 есть, а метаданных/вариантов в БД нет.
|
||||
Удаляем только если объект старше ORPHAN_GRACE_SECONDS.
|
||||
- Missing: метаданные есть, объекта в S3 нет — warning.
|
||||
- Staging: объекты staging/ старше grace удаляются (не привязаны к текущей работе).
|
||||
- Также периодически возвращает зависшие processing в queued.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from . import s3
|
||||
from .db import connection, init_pool, close_pool
|
||||
from .postgres import storage
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
log = logging.getLogger("s3_cleanup")
|
||||
|
||||
|
||||
def _interval() -> int:
|
||||
try:
|
||||
return max(30, int(os.environ.get("CLEANUP_INTERVAL_SECONDS", "300")))
|
||||
except ValueError:
|
||||
return 300
|
||||
|
||||
|
||||
def _grace() -> int:
|
||||
try:
|
||||
return max(0, int(os.environ.get("ORPHAN_GRACE_SECONDS", "3600")))
|
||||
except ValueError:
|
||||
return 3600
|
||||
|
||||
|
||||
def _known_keys() -> set[str]:
|
||||
"""Все ожидаемые object key: оригиналы + варианты."""
|
||||
keys: set[str] = set()
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT id, chat_id, object_key FROM attachments")
|
||||
for aid, cid, okey in cur.fetchall():
|
||||
keys.add(okey)
|
||||
keys.add(s3.original_key(cid, aid))
|
||||
cur.execute("SELECT attachment_id, variant FROM attachment_variants")
|
||||
# нужен chat_id для variant_key
|
||||
# variants: отдельным запросом с join
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT a.chat_id, v.attachment_id, v.variant"
|
||||
" FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id"
|
||||
)
|
||||
for cid, aid, variant in cur.fetchall():
|
||||
keys.add(s3.variant_key(cid, aid, variant))
|
||||
return keys
|
||||
|
||||
|
||||
def _active_attachment_ids() -> set[str]:
|
||||
"""ID вложений в статусах queued/processing — их staging трогать нельзя."""
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id FROM attachments WHERE status IN ('queued', 'processing')"
|
||||
)
|
||||
return {str(r[0]) for r in cur.fetchall()}
|
||||
|
||||
|
||||
def _run_once() -> dict:
|
||||
now = datetime.now(timezone.utc)
|
||||
grace_cutoff = now - timedelta(seconds=_grace())
|
||||
|
||||
known = _known_keys()
|
||||
active_ids = _active_attachment_ids()
|
||||
orphan_deleted = 0
|
||||
missing = 0
|
||||
scanned = 0
|
||||
staging_deleted = 0
|
||||
|
||||
# attachments/ — orphan по known
|
||||
for obj in s3.list_objects("attachments/"):
|
||||
scanned += 1
|
||||
key = obj["Key"]
|
||||
if key in known:
|
||||
continue
|
||||
# Orphan: в S3 есть, в БД нет. Удаляем только старше grace period.
|
||||
lm = obj.get("LastModified")
|
||||
if lm is None:
|
||||
continue
|
||||
if lm.tzinfo is None:
|
||||
lm = lm.replace(tzinfo=timezone.utc)
|
||||
if lm > grace_cutoff:
|
||||
# Свежий объект — возможно in-flight upload; не трогаем.
|
||||
continue
|
||||
try:
|
||||
s3.delete_object(key)
|
||||
orphan_deleted += 1
|
||||
log.info("event=orphan_deleted key=%s", key)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=orphan_delete_failed key=%s type=%s",
|
||||
key, type(exc).__name__)
|
||||
|
||||
# staging/ — удаляем только старое И не принадлежащее активной задаче.
|
||||
# Ключ: staging/<chat_id>/<attachment_id>/original
|
||||
for obj in s3.list_objects("staging/"):
|
||||
key = obj["Key"]
|
||||
parts = key.split("/")
|
||||
# Ожидаем минимум 4 части: staging / chat_id / attachment_id / original
|
||||
if len(parts) < 4 or parts[0] != "staging":
|
||||
continue
|
||||
attachment_id = parts[2]
|
||||
if attachment_id in active_ids:
|
||||
# Задача ещё в queued/processing — staging нужен worker'у.
|
||||
continue
|
||||
lm = obj.get("LastModified")
|
||||
if lm is None:
|
||||
continue
|
||||
if lm.tzinfo is None:
|
||||
lm = lm.replace(tzinfo=timezone.utc)
|
||||
if lm > grace_cutoff:
|
||||
continue
|
||||
try:
|
||||
s3.delete_object(key)
|
||||
staging_deleted += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# missing
|
||||
for key in known:
|
||||
if s3.head_object(key) is None:
|
||||
missing += 1
|
||||
log.warning("event=missing_object key=%s", key)
|
||||
|
||||
# зависшие processing → queued
|
||||
try:
|
||||
recovered = storage.recover_stuck_processing()
|
||||
if recovered:
|
||||
log.info("event=stuck_processing_recovered count=%s", recovered)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=recover_stuck_failed type=%s", type(exc).__name__)
|
||||
|
||||
return {
|
||||
"scanned": scanned,
|
||||
"orphan_deleted": orphan_deleted,
|
||||
"missing": missing,
|
||||
"known": len(known),
|
||||
"staging_deleted": staging_deleted,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_pool()
|
||||
interval = _interval()
|
||||
log.info("event=cleanup_start interval_s=%s grace_s=%s", interval, _grace())
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
stats = _run_once()
|
||||
log.info(
|
||||
"event=cleanup_pass scanned=%s known=%s orphan_deleted=%s"
|
||||
" missing=%s staging_deleted=%s",
|
||||
stats["scanned"], stats["known"], stats["orphan_deleted"],
|
||||
stats["missing"], stats["staging_deleted"],
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=cleanup_failed type=%s", type(exc).__name__)
|
||||
time.sleep(interval)
|
||||
finally:
|
||||
close_pool()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Хранилище сессий. Redis (основной режим, lab2 level 4) или PostgreSQL (fallback).
|
||||
|
||||
Ключевое: при недоступности Redis поднимаем RuntimeError("redis_unavailable"),
|
||||
чтобы main.py отдал 503 unavailable, а не 500 internal_error.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
from ..models.schemas import User
|
||||
from . import db as _db
|
||||
|
||||
try:
|
||||
import redis as _redis
|
||||
except ImportError: # pragma: no cover
|
||||
_redis = None
|
||||
|
||||
|
||||
_SESSION_PREFIX = "sess:"
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _store() -> str:
|
||||
return os.environ.get("SESSION_STORE", "redis").strip().lower()
|
||||
|
||||
|
||||
def _redis_client():
|
||||
"""Возвращает подключённый клиент Redis или поднимает RuntimeError.
|
||||
|
||||
RuntimeError("redis_unavailable: ...") ловит main.runtime_error_handler → 503.
|
||||
"""
|
||||
if _redis is None:
|
||||
raise RuntimeError("redis_unavailable: redis library is not installed")
|
||||
url = os.environ.get("REDIS_URL", "redis://redis:6379/0")
|
||||
try:
|
||||
client = _redis.from_url(
|
||||
url,
|
||||
decode_responses=True,
|
||||
socket_timeout=2.0,
|
||||
socket_connect_timeout=2.0,
|
||||
)
|
||||
client.ping()
|
||||
return client
|
||||
except _redis.RedisError as exc:
|
||||
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
|
||||
|
||||
|
||||
def _user_key(user_id: UUID) -> str:
|
||||
return str(user_id)
|
||||
|
||||
|
||||
def create_session(user_id: UUID, ttl_seconds: int) -> tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = now_utc() + timedelta(seconds=ttl_seconds)
|
||||
if _store() == "redis":
|
||||
r = _redis_client()
|
||||
payload = json.dumps({"user_id": _user_key(user_id), "expires_at": expires_at.isoformat()})
|
||||
try:
|
||||
# SETEX: TTL — сама суть Redis-сессий
|
||||
r.setex(_SESSION_PREFIX + token, ttl_seconds, payload)
|
||||
except _redis.RedisError as exc:
|
||||
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
|
||||
else:
|
||||
with _db.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO sessions (token, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token, user_id, expires_at),
|
||||
)
|
||||
conn.commit()
|
||||
return token, expires_at
|
||||
|
||||
|
||||
def get_session_user(token: str) -> Optional[User]:
|
||||
if _store() == "redis":
|
||||
r = _redis_client()
|
||||
try:
|
||||
raw = r.get(_SESSION_PREFIX + token)
|
||||
except _redis.RedisError as exc:
|
||||
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
user_id = UUID(data["user_id"])
|
||||
expires_at = datetime.fromisoformat(data["expires_at"])
|
||||
except Exception:
|
||||
return None
|
||||
if expires_at <= now_utc():
|
||||
try:
|
||||
r.delete(_SESSION_PREFIX + token)
|
||||
except _redis.RedisError:
|
||||
pass
|
||||
return None
|
||||
return _load_user(user_id)
|
||||
# postgres fallback
|
||||
with _db.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
|
||||
" FROM sessions s JOIN users u ON u.id = s.user_id"
|
||||
" WHERE s.token = %s",
|
||||
(token,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
if row[4] <= now_utc():
|
||||
cur.execute("DELETE FROM sessions WHERE token = %s", (token,))
|
||||
conn.commit()
|
||||
return None
|
||||
return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3])
|
||||
|
||||
|
||||
def revoke_session(token: str) -> None:
|
||||
if _store() == "redis":
|
||||
r = _redis_client()
|
||||
try:
|
||||
r.delete(_SESSION_PREFIX + token)
|
||||
except _redis.RedisError as exc:
|
||||
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
|
||||
return
|
||||
with _db.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM sessions WHERE token = %s", (token,))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _load_user(user_id: UUID) -> Optional[User]:
|
||||
with _db.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3]) if row else None
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Единая точка инициализации OpenTelemetry для API и worker.
|
||||
|
||||
- OTLP/HTTP → Collector (otel-collector:4318).
|
||||
- resource: service.name/version/instance.id/deployment.environment.name=lab.
|
||||
- LoggingFilter подставляет trace_id/span_id в каждую запись, чтобы они
|
||||
попадали и в stdout (видно глазами), и в Loki (в тексте строки) — это
|
||||
требование lab6 level4 (связь лога и trace).
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
|
||||
_INITIALIZED = False
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or uuid.uuid4().hex[:12]
|
||||
|
||||
|
||||
def _endpoint() -> str:
|
||||
return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
|
||||
|
||||
def _resource(service_name: str, service_version: str) -> Resource:
|
||||
return Resource.create(
|
||||
{
|
||||
"service.name": service_name,
|
||||
"service.version": service_version,
|
||||
"service.instance.id": INSTANCE_ID,
|
||||
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _otel_cert_kwargs() -> dict:
|
||||
"""mTLS к Collector (lab7): server CA + client identity (api/worker)."""
|
||||
kwargs = {}
|
||||
server_ca = os.environ.get("OTEL_EXPORTER_OTLP_CERTIFICATE", "/certs/ca.crt")
|
||||
client_cert = os.environ.get("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", "")
|
||||
client_key = os.environ.get("OTEL_EXPORTER_OTLP_CLIENT_KEY", "")
|
||||
if server_ca and os.path.exists(server_ca):
|
||||
kwargs["certificate_file"] = server_ca
|
||||
if client_cert and os.path.exists(client_cert):
|
||||
kwargs["client_certificate_file"] = client_cert
|
||||
if client_key and os.path.exists(client_key):
|
||||
kwargs["client_key_file"] = client_key
|
||||
return kwargs
|
||||
|
||||
|
||||
def _build_trace_provider(resource: Resource) -> TracerProvider:
|
||||
provider = TracerProvider(resource=resource)
|
||||
exporter = OTLPSpanExporter(
|
||||
endpoint=_endpoint() + "/v1/traces", timeout=5, **_otel_cert_kwargs()
|
||||
)
|
||||
provider.add_span_processor(
|
||||
BatchSpanProcessor(exporter, max_queue_size=2048,
|
||||
max_export_batch_size=512,
|
||||
schedule_delay_millis=2000)
|
||||
)
|
||||
return provider
|
||||
|
||||
|
||||
def _build_meter_provider(resource: Resource) -> MeterProvider:
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(
|
||||
endpoint=_endpoint() + "/v1/metrics", timeout=5, **_otel_cert_kwargs()
|
||||
),
|
||||
export_interval_millis=5000,
|
||||
)
|
||||
return MeterProvider(resource=resource, metric_readers=[reader])
|
||||
|
||||
|
||||
def _build_logger_provider(resource: Resource) -> LoggerProvider:
|
||||
provider = LoggerProvider(resource=resource)
|
||||
provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(
|
||||
endpoint=_endpoint() + "/v1/logs", timeout=5, **_otel_cert_kwargs()
|
||||
),
|
||||
max_queue_size=2048,
|
||||
max_export_batch_size=512,
|
||||
schedule_delay_millis=2000,
|
||||
)
|
||||
)
|
||||
return provider
|
||||
|
||||
|
||||
class TraceContextFilter(logging.Filter):
|
||||
"""Подставляет record.trace_id / record.span_id из активного span.
|
||||
|
||||
Если активного span нет — пишет "0"*32 и "0"*16, чтобы лог-формат
|
||||
оставался валидным (не падал с KeyError). Так же OTel сам поступает.
|
||||
"""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
span = trace.get_current_span()
|
||||
span_ctx = span.get_span_context() if span else None
|
||||
if span_ctx is not None and span_ctx.is_valid:
|
||||
record.trace_id = format(span_ctx.trace_id, "032x")
|
||||
record.span_id = format(span_ctx.span_id, "016x")
|
||||
else:
|
||||
record.trace_id = "0" * 32
|
||||
record.span_id = "0" * 16
|
||||
return True
|
||||
|
||||
|
||||
_LOG_FORMAT = (
|
||||
"%(asctime)s %(levelname)s %(name)s "
|
||||
"trace_id=%(trace_id)s span_id=%(span_id)s %(message)s"
|
||||
)
|
||||
|
||||
|
||||
def install_log_format() -> None:
|
||||
"""Единый формат логов API/worker/outbox.
|
||||
|
||||
Поля trace_id/span_id гарантированно существуют благодаря
|
||||
TraceContextFilter. Их наличие обеспечивает уровень 4 (корреляция
|
||||
логов и trace) и делает Loki-запросы `|= "trace_id="` рабочими.
|
||||
"""
|
||||
handler = logging.StreamHandler()
|
||||
handler.addFilter(TraceContextFilter())
|
||||
handler.setFormatter(logging.Formatter(_LOG_FORMAT))
|
||||
root = logging.getLogger()
|
||||
root.handlers = [handler]
|
||||
root.setLevel(logging.INFO)
|
||||
|
||||
|
||||
def init_telemetry(service_name: str, service_version: str = "1.6.0") -> None:
|
||||
"""Идемпотентная инициализация OTel для процесса."""
|
||||
global _INITIALIZED
|
||||
if _INITIALIZED:
|
||||
return
|
||||
|
||||
# Сначала — единый формат логов и фильтр trace_id.
|
||||
install_log_format()
|
||||
|
||||
resource = _resource(service_name, service_version)
|
||||
trace.set_tracer_provider(_build_trace_provider(resource))
|
||||
metrics.set_meter_provider(_build_meter_provider(resource))
|
||||
logger_provider = _build_logger_provider(resource)
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
# OTel handler шлёт те же записи в Collector → Loki. Он тоже видит
|
||||
# trace_id из record, потому что span-контекст един.
|
||||
#
|
||||
# ВАЖНО: LoggingHandler.emit() формирует тело через self.format(record),
|
||||
# где self.formatter наследуется от logging.Handler. Без явного
|
||||
# formatter'а тело будет голым сообщением (msg % args) — без
|
||||
# trace_id=/span_id=. Это ломало поиск в Loki `|= "trace_id="`.
|
||||
# Поэтому задаём тот же формат и навешиваем TraceContextFilter
|
||||
# непосредственно на OTel-хэндлер: фильтр должен отработать ДО emit,
|
||||
# чтобы record.trace_id существовал на момент форматирования.
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otel_handler.addFilter(TraceContextFilter())
|
||||
otel_handler.setFormatter(logging.Formatter(_LOG_FORMAT))
|
||||
logging.getLogger().addHandler(otel_handler)
|
||||
|
||||
_INITIALIZED = True
|
||||
logging.getLogger("telemetry").info(
|
||||
"event=otel_init service=%s instance_id=%s endpoint=%s",
|
||||
service_name, INSTANCE_ID, _endpoint(),
|
||||
)
|
||||
|
||||
|
||||
def tracer(name: str = "app"):
|
||||
return trace.get_tracer(name)
|
||||
|
||||
|
||||
def meter(name: str = "app"):
|
||||
return metrics.get_meter(name)
|
||||
@@ -0,0 +1,405 @@
|
||||
"""Worker lab5/lab6: обработка вложений из RabbitMQ с трейсингом.
|
||||
|
||||
- at-least-once, идемпотентно, retry через TTL-очередь, DLQ.
|
||||
- OTel: consumer span на каждый job, извлечение trace context из заголовков
|
||||
сообщения (W3C traceparent/tracestate), child spans на S3/DB операции.
|
||||
- Метрики: jobs_processed_total{outcome}, job_processing_duration_seconds.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pika
|
||||
from PIL import Image, ImageOps
|
||||
from opentelemetry import context as otel_context, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
from .storage import s3
|
||||
from .storage.db import init_pool, close_pool
|
||||
from .storage.postgres import storage
|
||||
from .storage.rabbit import (
|
||||
QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology,
|
||||
publish_retry, publish_dlq,
|
||||
)
|
||||
from .telemetry import init_telemetry, meter
|
||||
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
||||
log = logging.getLogger("worker")
|
||||
|
||||
PIPELINE_VERSION = 1
|
||||
LEASE_SECONDS = 60
|
||||
THUMB_SIZE = (256, 256)
|
||||
CROP_SIZE = (128, 128)
|
||||
OPT_SIZE = (1280, 1280)
|
||||
OPT_QUALITY = 82
|
||||
|
||||
_stop = False
|
||||
_propagator = TraceContextTextMapPropagator()
|
||||
|
||||
|
||||
# OTel инициализация — до первого сообщения
|
||||
init_telemetry("messenger-worker", "1.6.0")
|
||||
_tracer = trace.get_tracer("worker")
|
||||
_meter = meter("messenger-worker")
|
||||
_jobs_counter = _meter.create_counter(
|
||||
"jobs_processed_total", description="Jobs processed by outcome", unit="1",
|
||||
)
|
||||
# Явные границы бакетов в СЕКУНДАХ: job-обработка (decode + resize + 3×S3 PUT
|
||||
# + DB) обычно от десятков мс до единиц секунд. Дефолтные экспоненциальные
|
||||
# бакеты слишком грубы для p95/p99 на этой шкале.
|
||||
_JOB_DURATION_BUCKETS = [
|
||||
0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5,
|
||||
0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0,
|
||||
]
|
||||
_jobs_hist = _meter.create_histogram(
|
||||
"job_processing_duration_seconds",
|
||||
description="Job processing duration",
|
||||
unit="s",
|
||||
explicit_bucket_boundaries_advisory=_JOB_DURATION_BUCKETS,
|
||||
)
|
||||
|
||||
|
||||
# --- span helpers: дочерние операции S3/DB ---
|
||||
#
|
||||
# ВАЖНО: атрибуты, содержащие точку (`db.operation`, `aws.s3.key` и т.п.),
|
||||
# нельзя передавать как kwargs — Python-идентификатор с точкой невозможен.
|
||||
# Поэтому оба хелпера принимают dict атрибутов.
|
||||
|
||||
def _s3_span(name: str, attrs: dict):
|
||||
"""Context manager для S3-операции (CLIENT span внутри job.consume)."""
|
||||
return _tracer.start_as_current_span(
|
||||
name,
|
||||
kind=SpanKind.CLIENT,
|
||||
attributes={"messaging.system": "s3", **attrs},
|
||||
)
|
||||
|
||||
|
||||
def _db_span(name: str, attrs: dict):
|
||||
"""Context manager для DB-операции (CLIENT span внутри job.consume)."""
|
||||
return _tracer.start_as_current_span(
|
||||
name,
|
||||
kind=SpanKind.CLIENT,
|
||||
attributes={"db.system": "postgresql", **attrs},
|
||||
)
|
||||
|
||||
|
||||
def _handle_sigterm(signum, frame):
|
||||
global _stop
|
||||
log.info("event=sigterm_received signal=%s", signum)
|
||||
_stop = True
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _to_rgb_white(img: Image.Image) -> Image.Image:
|
||||
img = ImageOps.exif_transpose(img)
|
||||
if img.mode in ("RGBA", "LA", "P"):
|
||||
img = img.convert("RGBA")
|
||||
bg = Image.new("RGB", img.size, (255, 255, 255))
|
||||
bg.paste(img, mask=img.split()[-1])
|
||||
return bg
|
||||
if img.mode != "RGB":
|
||||
return img.convert("RGB")
|
||||
return img
|
||||
|
||||
|
||||
def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]:
|
||||
w, h = img.size
|
||||
scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0)
|
||||
nw = max(1, int(w * scale))
|
||||
nh = max(1, int(h * scale))
|
||||
thumb = img.resize((nw, nh), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
thumb.save(buf, format="JPEG", quality=88, optimize=True)
|
||||
return thumb, buf.getvalue()
|
||||
|
||||
|
||||
def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]:
|
||||
w, h = img.size
|
||||
side = min(w, h)
|
||||
left = (w - side) // 2
|
||||
top = (h - side) // 2
|
||||
sq = img.crop((left, top, left + side, top + side))
|
||||
crop = sq.resize(CROP_SIZE, Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
crop.save(buf, format="JPEG", quality=88, optimize=True)
|
||||
return crop, buf.getvalue()
|
||||
|
||||
|
||||
def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]:
|
||||
w, h = img.size
|
||||
scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0)
|
||||
nw = max(1, int(w * scale))
|
||||
nh = max(1, int(h * scale))
|
||||
opt = img.resize((nw, nh), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
|
||||
return opt, buf.getvalue()
|
||||
|
||||
|
||||
def _sha256(data: bytes) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _s3_put_variant(chat_id, att_id, name: str, data: bytes, ctype: str) -> str:
|
||||
"""Общий путь: S3 PUT варианта внутри CLIENT span."""
|
||||
key = s3.variant_key(chat_id, att_id, name)
|
||||
with _s3_span("s3.put_object", {
|
||||
"aws.s3.key": key,
|
||||
"s3.operation": "put_object",
|
||||
"s3.variant": name,
|
||||
"s3.content_type": ctype,
|
||||
"s3.size_bytes": len(data),
|
||||
}):
|
||||
s3.put_bytes(key, data, ctype)
|
||||
return key
|
||||
|
||||
|
||||
def process_job(payload: dict) -> None:
|
||||
"""Обработка одного job. Идемпотентна через finalize_ready.
|
||||
|
||||
Внутри активного job.consume (CONSUMER) span создаются дочерние CLIENT
|
||||
spans для S3 и DB, чтобы разложить задержку обработки по зависимостям.
|
||||
"""
|
||||
attachment_id = payload["attachment_id"]
|
||||
pipeline_version = payload.get("pipeline_version", 1)
|
||||
job_id = payload["job_id"]
|
||||
|
||||
log.info("event=job_received job_id=%s attachment_id=%s attempt=%s",
|
||||
job_id, attachment_id, payload.get("attempt", 1))
|
||||
|
||||
with _db_span("db.claim_processing", {
|
||||
"db.operation": "UPDATE",
|
||||
"attachment_id": str(attachment_id),
|
||||
}):
|
||||
claimed = storage.claim_processing(attachment_id, LEASE_SECONDS)
|
||||
if not claimed:
|
||||
log.info("event=job_skipped_already_done attachment_id=%s", attachment_id)
|
||||
return
|
||||
|
||||
pair = storage.get_attachment_with_key(attachment_id)
|
||||
if pair is None:
|
||||
log.warning("event=attachment_missing attachment_id=%s", attachment_id)
|
||||
return
|
||||
att, staging_key = pair
|
||||
if att.status == "ready":
|
||||
log.info("event=already_ready attachment_id=%s", attachment_id)
|
||||
return
|
||||
|
||||
try:
|
||||
with _s3_span("s3.get_object", {
|
||||
"aws.s3.key": staging_key,
|
||||
"s3.operation": "get_object",
|
||||
"s3.variant": "staging",
|
||||
}):
|
||||
raw, ctype = s3.get_object(staging_key)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise RuntimeError("staging_unavailable") from exc
|
||||
|
||||
try:
|
||||
img = Image.open(io.BytesIO(raw))
|
||||
img.load()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
with _db_span("db.finalize_failed", {
|
||||
"db.operation": "UPDATE",
|
||||
"error_code": "invalid_image",
|
||||
}):
|
||||
storage.finalize_failed(attachment_id, "invalid_image")
|
||||
log.warning("event=invalid_image attachment_id=%s type=%s",
|
||||
attachment_id, type(exc).__name__)
|
||||
return
|
||||
|
||||
img_rgb = _to_rgb_white(img)
|
||||
|
||||
# Финал: S3 PUT оригинала
|
||||
final_key = s3.original_key(att.chat_id, att.id)
|
||||
with _s3_span("s3.put_object", {
|
||||
"aws.s3.key": final_key,
|
||||
"s3.operation": "put_object",
|
||||
"s3.variant": "original",
|
||||
"s3.content_type": att.content_type,
|
||||
"s3.size_bytes": len(raw),
|
||||
}):
|
||||
s3.put_bytes(final_key, raw, att.content_type)
|
||||
|
||||
variants = [{
|
||||
"name": "original",
|
||||
"object_key": final_key,
|
||||
"content_type": att.content_type,
|
||||
"size_bytes": len(raw),
|
||||
"width": img.width,
|
||||
"height": img.height,
|
||||
"sha256": _sha256(raw),
|
||||
}]
|
||||
|
||||
# Thumbnail
|
||||
_, thumb_bytes = _make_thumbnail(img_rgb)
|
||||
thumb_key = _s3_put_variant(att.chat_id, att.id, "thumbnail", thumb_bytes, "image/jpeg")
|
||||
tw, th = Image.open(io.BytesIO(thumb_bytes)).size
|
||||
variants.append({
|
||||
"name": "thumbnail", "object_key": thumb_key,
|
||||
"content_type": "image/jpeg", "size_bytes": len(thumb_bytes),
|
||||
"width": tw, "height": th, "sha256": _sha256(thumb_bytes),
|
||||
})
|
||||
|
||||
# Crop
|
||||
_, crop_bytes = _make_crop(img_rgb)
|
||||
crop_key = _s3_put_variant(att.chat_id, att.id, "crop", crop_bytes, "image/jpeg")
|
||||
cw, ch = Image.open(io.BytesIO(crop_bytes)).size
|
||||
variants.append({
|
||||
"name": "crop", "object_key": crop_key,
|
||||
"content_type": "image/jpeg", "size_bytes": len(crop_bytes),
|
||||
"width": cw, "height": ch, "sha256": _sha256(crop_bytes),
|
||||
})
|
||||
|
||||
# Optimized
|
||||
_, opt_bytes = _make_optimized(img_rgb)
|
||||
opt_key = _s3_put_variant(att.chat_id, att.id, "optimized", opt_bytes, "image/webp")
|
||||
ow, oh = Image.open(io.BytesIO(opt_bytes)).size
|
||||
variants.append({
|
||||
"name": "optimized", "object_key": opt_key,
|
||||
"content_type": "image/webp", "size_bytes": len(opt_bytes),
|
||||
"width": ow, "height": oh, "sha256": _sha256(opt_bytes),
|
||||
})
|
||||
|
||||
with _db_span("db.finalize_ready", {
|
||||
"db.operation": "UPDATE",
|
||||
"pipeline_version": pipeline_version,
|
||||
"variants_count": len(variants),
|
||||
}):
|
||||
storage.finalize_ready(attachment_id, pipeline_version, variants)
|
||||
|
||||
# Cleanup staging — best-effort, не критичен для результата.
|
||||
try:
|
||||
with _s3_span("s3.delete_object", {
|
||||
"aws.s3.key": staging_key,
|
||||
"s3.operation": "delete_object",
|
||||
"s3.variant": "staging",
|
||||
}):
|
||||
s3.delete_object(staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
log.info("event=job_ready attachment_id=%s job_id=%s variants=%d",
|
||||
attachment_id, job_id, len(variants))
|
||||
|
||||
|
||||
def _extract_trace_context(properties) -> otel_context.Context:
|
||||
"""Извлекает W3C traceparent/tracestate из headers. Ошибка → пустой контекст."""
|
||||
try:
|
||||
headers = dict(properties.headers or {})
|
||||
carrier = {
|
||||
k.lower(): str(v)
|
||||
for k, v in headers.items()
|
||||
if k.lower() in ("traceparent", "tracestate")
|
||||
}
|
||||
if not carrier:
|
||||
return otel_context.Context()
|
||||
return _propagator.extract(carrier=carrier)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("event=trace_extract_failed type=%s", type(exc).__name__)
|
||||
return otel_context.Context()
|
||||
|
||||
|
||||
def _on_message(ch, method, properties, body):
|
||||
"""Callback consumer. ack/nack — явно. Оборачивает обработку в CONSUMER span."""
|
||||
parent_ctx = _extract_trace_context(properties)
|
||||
started = time.monotonic()
|
||||
with _tracer.start_as_current_span(
|
||||
"job.consume", context=parent_ctx, kind=SpanKind.CONSUMER,
|
||||
attributes={"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE},
|
||||
) as span:
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except Exception:
|
||||
log.error("event=invalid_job_json")
|
||||
_jobs_counter.add(1, {"job.type": "unknown", "outcome": "invalid"})
|
||||
ch.basic_nack(method.delivery_tag, requeue=False)
|
||||
return
|
||||
|
||||
headers = dict(properties.headers or {})
|
||||
retry_count = int(headers.get("x-retry-count", 0))
|
||||
payload["attempt"] = retry_count + 1
|
||||
payload["_headers"] = headers
|
||||
span.set_attribute("job.id", str(payload.get("job_id", "")))
|
||||
span.set_attribute("job.type", str(payload.get("type", "unknown")))
|
||||
span.set_attribute("job.attempt", retry_count + 1)
|
||||
|
||||
try:
|
||||
process_job(payload)
|
||||
duration = time.monotonic() - started
|
||||
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "success"})
|
||||
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "success"})
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
duration = time.monotonic() - started
|
||||
span.record_exception(exc)
|
||||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
log.warning("event=job_retry attachment_id=%s attempt=%d type=%s",
|
||||
payload.get("attachment_id"), retry_count + 1, type(exc).__name__)
|
||||
if retry_count + 1 >= MAX_ATTEMPTS:
|
||||
try:
|
||||
with _db_span("db.finalize_failed", {
|
||||
"db.operation": "UPDATE",
|
||||
"error_code": "retry_exhausted",
|
||||
}):
|
||||
storage.finalize_failed(payload["attachment_id"], "retry_exhausted")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
publish_dlq({k: v for k, v in payload.items() if k != "_headers"})
|
||||
except Exception:
|
||||
pass
|
||||
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "failed"})
|
||||
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "failed"})
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
return
|
||||
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "retry"})
|
||||
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "retry"})
|
||||
try:
|
||||
publish_retry(payload, retry_count + 1)
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
except Exception:
|
||||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||
init_pool()
|
||||
log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS)
|
||||
|
||||
while not _stop:
|
||||
try:
|
||||
conn = connect()
|
||||
ch = conn.channel()
|
||||
setup_topology(ch)
|
||||
ch.basic_qos(prefetch_count=1)
|
||||
ch.basic_consume(queue=QUEUE, on_message_callback=_on_message)
|
||||
log.info("event=consumer_started")
|
||||
|
||||
while not _stop and conn.is_open:
|
||||
conn.process_data_events(time_limit=1)
|
||||
|
||||
ch.stop_consuming()
|
||||
conn.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=worker_loop_error type=%s", type(exc).__name__)
|
||||
time.sleep(3)
|
||||
|
||||
close_pool()
|
||||
log.info("event=worker_stop")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Единая команда запуска собственных тестов.
|
||||
Запускает: test_negative.py, test_concurrency.py.
|
||||
Публичный smoke запускается отдельно."""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def run_one(script, base_url):
|
||||
print("=== %s ===" % script.name)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(script), "--base-url", base_url],
|
||||
cwd=str(HERE),
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print("FAIL: %s" % script.name)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="Lab1 student tests (level 4)")
|
||||
p.add_argument("--base-url", default="http://localhost:8080")
|
||||
args = p.parse_args()
|
||||
|
||||
ok = True
|
||||
for script in ("lab1_test_negative.py", "lab1_test_concurrency.py"):
|
||||
if not run_one(HERE / script, args.base_url):
|
||||
ok = False
|
||||
if not ok:
|
||||
raise SystemExit(1)
|
||||
print("PASS: all student tests (level 4) passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Конкурентный тест: 100 сообщений от 10 клиентов -> ровно 100 уникальных.
|
||||
Проверяет защиту in-memory структур от гонок (RLock)."""
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
TOTAL_MESSAGES = 100
|
||||
CLIENTS = 10
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab1 concurrency safety")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"]
|
||||
|
||||
# Каждый «клиент» — свой actor (alice/bob) для реалистичной конкуренции
|
||||
actors = [alice, bob]
|
||||
barrier = threading.Barrier(CLIENTS)
|
||||
|
||||
def worker(worker_id):
|
||||
local = Client(args.base_url, args.ca_file, timeout=15)
|
||||
actor = actors[worker_id % len(actors)]
|
||||
barrier.wait() # синхронный старт всех потоков
|
||||
produced = []
|
||||
for i in range(TOTAL_MESSAGES // CLIENTS):
|
||||
text = "c-%02d-%02d-%s" % (worker_id, i, uuid.uuid4().hex[:8])
|
||||
msg = local.message(actor, chat, text)["json"]
|
||||
produced.append(msg)
|
||||
return produced
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
results = list(pool.map(worker, range(CLIENTS)))
|
||||
|
||||
all_ids = [m["id"] for chunk in results for m in chunk]
|
||||
check(len(all_ids) == TOTAL_MESSAGES,
|
||||
"expected %d message ids, got %d" % (TOTAL_MESSAGES, len(all_ids)))
|
||||
check(len(set(all_ids)) == TOTAL_MESSAGES,
|
||||
"duplicate message ids under concurrency: %d unique" % len(set(all_ids)))
|
||||
|
||||
# Читаем всю историю через пагинацию
|
||||
collected = []
|
||||
cursor = None
|
||||
from urllib.parse import quote
|
||||
for _ in range(50):
|
||||
q = path + "/messages?limit=100"
|
||||
if cursor:
|
||||
q += "&cursor=" + quote(cursor, safe="")
|
||||
page = c.call("GET", q, actor=alice, schema="MessagePage")["json"]
|
||||
collected.extend(m["id"] for m in page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
check(len(collected) == TOTAL_MESSAGES,
|
||||
"stored messages: %d, expected %d" % (len(collected), TOTAL_MESSAGES))
|
||||
check(len(set(collected)) == TOTAL_MESSAGES,
|
||||
"stored messages contain duplicates: %d unique" % len(set(collected)))
|
||||
|
||||
print("PASS: lab1 concurrency: %d messages from %d clients, "
|
||||
"%d unique stored and readable" % (TOTAL_MESSAGES, CLIENTS, len(set(collected))))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 1 (оценка 5). Предел in-memory архитектуры.
|
||||
|
||||
Гипотеза: на больших историях список сообщений при чтении каждый раз
|
||||
копируется и сортируется под RLock, что ухудшает p95/p99 при росте
|
||||
истории. Убираем сортировку на чтении (append сохраняет порядок) и
|
||||
сравниваем "до/после" при тех же лимитах.
|
||||
|
||||
Эксперимент: >=3 размера истории (H) x >=3 уровня конкуренции (C).
|
||||
Метрики: successful RPS, доля ошибок, p50/p95/p99 только успешных
|
||||
запросов, RSS процесса API (читается через docker stats или /proc).
|
||||
|
||||
Скрипт не меняет API; использует те же публичные клиенты из tests/.
|
||||
Stdlib only, Python 3.9+.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
|
||||
# --- профиль эксперимента -------------------------------------------------
|
||||
HISTORY_SIZES = [100, 1000, 5000] # >=3 размера истории
|
||||
CONCURRENCY_LEVELS = [1, 4, 16] # >=3 уровня конкуренции
|
||||
DURATION_SECONDS = 10 # на каждую точку (H, C)
|
||||
API_CONTAINER = "mtusi-messenger-lab1" # имя контейнера для docker stats
|
||||
|
||||
|
||||
def rss_mb(container: str):
|
||||
"""RSS процесса API в МБ через docker stats --no-stream."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return None
|
||||
# Пример: "120.5MiB / 1GiB"
|
||||
used = out.stdout.strip().split("/")[0].strip()
|
||||
if used.endswith("MiB"):
|
||||
return round(float(used[:-3]), 2)
|
||||
if used.endswith("GiB"):
|
||||
return round(float(used[:-3]) * 1024, 2)
|
||||
if used.endswith("KiB"):
|
||||
return round(float(used[:-3]) / 1024, 2)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def seed_history(c, actor, chat, count, workers=16):
|
||||
"""Заполняет историю до count сообщений параллельно."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
def one(i):
|
||||
local = Client(c.base, timeout=15)
|
||||
local.message(actor, chat, "seed-%06d-%s" % (i, uuid.uuid4().hex[:6]))
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
list(pool.map(one, range(count)))
|
||||
|
||||
|
||||
def measure_read_point(c, chat, actor, duration, concurrency):
|
||||
"""Закрытый контур: параллельно читаем GET /messages?limit=100."""
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
barrier = threading.Barrier(concurrency)
|
||||
latencies, errors = [], 0
|
||||
errors_lock = threading.Lock()
|
||||
|
||||
def worker(_):
|
||||
nonlocal errors
|
||||
local = Client(c.base, timeout=15)
|
||||
barrier.wait()
|
||||
deadline = time.monotonic() + duration
|
||||
local_lat = []
|
||||
while time.monotonic() < deadline:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
local.call("GET", path, actor=actor, schema="MessagePage")
|
||||
local_lat.append(time.monotonic() - start)
|
||||
except Failure:
|
||||
with errors_lock:
|
||||
errors += 1
|
||||
return local_lat
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
results = list(pool.map(worker, range(concurrency)))
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
for chunk in results:
|
||||
latencies.extend(chunk)
|
||||
latencies.sort()
|
||||
|
||||
def pct(p):
|
||||
if not latencies:
|
||||
return None
|
||||
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
|
||||
|
||||
total = len(latencies) + errors
|
||||
return {
|
||||
"history_size": None, # заполнит вызывающий
|
||||
"concurrency": concurrency,
|
||||
"duration_s": round(elapsed, 3),
|
||||
"successful": len(latencies),
|
||||
"errors": errors,
|
||||
"error_ratio": round(errors / total, 6) if total else None,
|
||||
"rps": round(len(latencies) / elapsed, 3),
|
||||
"latency_ms": {"p50": pct(.50), "p95": pct(.95), "p99": pct(.99)},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Трек 1: предел in-memory архитектуры")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
|
||||
report = {"lab": 1, "track": 1, "profile": {
|
||||
"history_sizes": HISTORY_SIZES,
|
||||
"concurrency_levels": CONCURRENCY_LEVELS,
|
||||
"duration_seconds": DURATION_SECONDS,
|
||||
"note": "closed-loop; клиент ограничивает throughput; RSS — docker stats",
|
||||
}, "points": []}
|
||||
|
||||
for h in HISTORY_SIZES:
|
||||
# Досеиваем историю до h (bootstrap уже создал часть).
|
||||
# Считаем число сообщений в истории.
|
||||
# Предыдущая версия: limit=1 - возвращает только одну страницу и далее считает её длину страницы за длину всей истории,
|
||||
# Текущая версия: limit=100 - делаем полный обход истории по курсору => реальное число сообщений
|
||||
have = 0
|
||||
cursor = None
|
||||
from urllib.parse import quote
|
||||
while True:
|
||||
q = path + "?limit=100"
|
||||
if cursor:
|
||||
q += "&cursor=" + quote(cursor, safe="")
|
||||
page = c.call("GET", q, actor=alice, schema="MessagePage")["json"]
|
||||
have += len(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
if have < h:
|
||||
seed_history(c, alice, chat, h - have)
|
||||
|
||||
for con in CONCURRENCY_LEVELS:
|
||||
point = measure_read_point(c, chat, alice, DURATION_SECONDS, con)
|
||||
point["history_size"] = h
|
||||
point["rss_mb"] = rss_mb(API_CONTAINER)
|
||||
report["points"].append(point)
|
||||
print(json.dumps(point, ensure_ascii=False))
|
||||
|
||||
out = Path("evidence/lab1_track1_load.json")
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: track1 load matrix saved to", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Дополнительный CLI: --container для имени Docker-контейнера.
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(add_help=False)
|
||||
p.add_argument("--container", default=API_CONTAINER)
|
||||
extra, _ = p.parse_known_args()
|
||||
# Прокидываем в namespace аргументов arguments() через env-подобный трюк.
|
||||
sys.argv = [a for a in sys.argv if not a.startswith("--container")]
|
||||
import os
|
||||
os.environ.setdefault("TRACK1_CONTAINER", extra.container)
|
||||
|
||||
def _bind_container(ns):
|
||||
ns.container = os.environ.get("TRACK1_CONTAINER", API_CONTAINER)
|
||||
return ns
|
||||
|
||||
# Патчим arguments, чтобы добавить --container.
|
||||
import client as _client # noqa: E402
|
||||
_orig_arguments = _client.arguments
|
||||
|
||||
def _arguments(description, configure=None):
|
||||
def cfg(parser):
|
||||
if configure:
|
||||
configure(parser)
|
||||
parser.add_argument("--container", default=API_CONTAINER)
|
||||
return _orig_arguments(description, cfg)
|
||||
|
||||
_client.arguments = _arguments
|
||||
run(main)
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Негативные сценарии lab1: неверный JSON, чужой чат, пустое сообщение,
|
||||
дубликат username, обход нескольких страниц. Stdlib only, Python 3.9+."""
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
|
||||
def _raw_call(base_url, method, path, body=None, headers=None):
|
||||
"""Сырой запрос для проверки malformed JSON (client.py всегда сериализует)."""
|
||||
url = base_url.rstrip("/") + path
|
||||
data = body if isinstance(body, (bytes, type(None))) else body.encode("utf-8")
|
||||
req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return resp.status, resp.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read()
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab1 negative scenarios")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"]
|
||||
|
||||
# 1. Дубликат username -> 409
|
||||
c.call("POST", "/api/v1/users", 409, body=alice["registration"])
|
||||
|
||||
# 2. Неверный JSON (malformed) -> 400
|
||||
status, _ = _raw_call(
|
||||
c.base,
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body=b"{not json at all",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
check(status == 400, "malformed JSON must be 400, got %s" % status)
|
||||
|
||||
# 3. Лишние поля -> 400
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
body={"username": "extra_user_x", "display_name": "X", "unexpected": True},
|
||||
)
|
||||
|
||||
# 4. Неверный тип -> 400
|
||||
c.call("POST", "/api/v1/users", 400, body={"username": 123, "display_name": "X"})
|
||||
|
||||
# 5. Пустое / пробельное сообщение -> 400
|
||||
c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""})
|
||||
c.call("POST", path + "/messages", 400, actor=alice, body={"text": " "})
|
||||
|
||||
# 6. Чужой чат -> 404 (не 403)
|
||||
c.call("GET", path, 404, actor=eve)
|
||||
c.call("GET", path + "/messages", 404, actor=eve)
|
||||
c.call("POST", path + "/messages", 404, actor=eve, body={"text": "чужое"})
|
||||
|
||||
# 7. Пагинация: обход нескольких страниц без потерь и дублей
|
||||
texts = ["p-%02d" % i for i in range(7)]
|
||||
for t in texts:
|
||||
c.message(alice, chat, t)
|
||||
collected = []
|
||||
cursor = None
|
||||
seen_cursors = set()
|
||||
for _ in range(20): # предохранитель от бесконечного цикла
|
||||
q = path + "/messages?limit=2"
|
||||
if cursor:
|
||||
from urllib.parse import quote
|
||||
q += "&cursor=" + quote(cursor, safe="")
|
||||
page = c.call("GET", q, actor=bob, schema="MessagePage")["json"]
|
||||
collected.extend(m["text"] for m in page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
check(cursor not in seen_cursors, "cursor loop detected")
|
||||
seen_cursors.add(cursor)
|
||||
check(len(collected) == len(texts), "pagination lost messages: %d != %d" % (len(collected), len(texts)))
|
||||
check(len(set(collected)) == len(collected), "pagination duplicated messages")
|
||||
check(collected == texts, "pagination order/contents mismatch")
|
||||
|
||||
# 8. Некорректный limit -> 400
|
||||
c.call("GET", path + "/messages?limit=0", 400, actor=alice)
|
||||
c.call("GET", path + "/messages?limit=101", 400, actor=alice)
|
||||
|
||||
# 9. Неизвестный X-User-Id -> 401
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": str(uuid.uuid4())})
|
||||
|
||||
print("PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, "
|
||||
"wrong type, empty text, foreign chat, pagination, bad limit, unknown user)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,587 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4 lab2: expiry TTL, конкурентная регистрация, EXPLAIN, недоступность Redis/PostgreSQL.
|
||||
|
||||
Перед запуском:
|
||||
1) в .env выставить SESSION_TTL_SECONDS=5
|
||||
2) docker compose --env-file .env up -d --force-recreate api
|
||||
3) docker exec mtuci-redis redis-cli flushdb
|
||||
4) python student-tests/test_lab2_level4.py --base-url http://localhost:8080
|
||||
5) перед smoke/persistence вернуть SESSION_TTL_SECONDS=3600 и снова up -d --force-recreate api
|
||||
|
||||
Порядок тестов: тяжёлые по данным и конкурентные — ДО expiry-теста,
|
||||
потому что при коротком TTL любой долгий шаг съедает окно жизни сессии.
|
||||
Expiry-тест запускается последним и использует свежие сессии.
|
||||
|
||||
test_explain не зависит от TTL вообще: он сидирует данные напрямую
|
||||
через psql и проверяет план SQL-запроса, а не HTTP-слой.
|
||||
|
||||
test_chat_rollback проверяет транзакционную границу create_chat: он
|
||||
запускает патч-скрипт ВНУТРИ контейнера API (там есть psycopg и
|
||||
DATABASE_URL), monkey-patch-ит psycopg.Cursor.executemany так, чтобы
|
||||
INSERT в chat_members упал ПОСЛЕ успешного INSERT в chats. Затем через
|
||||
psql проверяется, что ни chat, ни chat_members не сохранились.
|
||||
"""
|
||||
import base64
|
||||
import datetime as dt
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name).
|
||||
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default")
|
||||
|
||||
# Имя контейнера API (зафиксировано в compose.yaml container_name).
|
||||
API_CONTAINER = os.environ.get("API_CONTAINER", "mtuci-messenger-lab2")
|
||||
|
||||
|
||||
# ---------- 1. Конкурентная регистрация одинакового username ----------
|
||||
def test_concurrent_registration(c):
|
||||
"""Один и тот же username с 8 потоков — ровно одна 201, остальные 409.
|
||||
|
||||
Проверяем ФАКТИЧЕСКИЕ HTTP-коды, а не факт исключения: timeout, 500
|
||||
или неверный формат ответа должны проваливать тест, а не выдаваться
|
||||
за ожидаемый конфликт. Дополнительно сверяем, что в БД создана
|
||||
ровно одна строка пользователя.
|
||||
"""
|
||||
username = "race_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
body = {"username": username, "display_name": "Race", "password": password}
|
||||
results = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def one(_):
|
||||
local = Client(c.base, timeout=10)
|
||||
try:
|
||||
r = local.call("POST", "/api/v1/users", (201, 409),
|
||||
body=body, schema="User" if True else None)
|
||||
# schema валидируется только для 2xx; при 409 приходит Error.
|
||||
status = r["status"]
|
||||
except Failure as exc:
|
||||
# Любая иная ошибка (timeout, 500, bad schema) — это не 409.
|
||||
status = "failure:%s" % exc
|
||||
with lock:
|
||||
results.append(status)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
list(pool.map(one, range(8)))
|
||||
|
||||
ok = [r for r in results if r == 201]
|
||||
conflict = [r for r in results if r == 409]
|
||||
bad = [r for r in results if r not in (201, 409)]
|
||||
check(not bad, "unexpected results under concurrent registration: %s" % results)
|
||||
check(len(ok) == 1, "expected exactly one 201, got %s" % results)
|
||||
check(len(conflict) == 7, "expected seven 409, got %s" % results)
|
||||
|
||||
# В БД должна быть ровно одна строка с этим username.
|
||||
dsn = _read_dsn()
|
||||
rows = _scalar(dsn, "SELECT count(*) FROM users WHERE username = '%s';" % username)
|
||||
check(int(rows) == 1,
|
||||
"ожидалась ровно одна строка users для %s, получено %s" % (username, rows))
|
||||
|
||||
|
||||
# ---------- 2. EXPLAIN на реальной БД ----------
|
||||
def _read_dsn() -> str:
|
||||
dsn = os.environ.get("DATABASE_URL")
|
||||
if not dsn:
|
||||
env_file = Path(__file__).resolve().parents[1] / ".env"
|
||||
if env_file.is_file():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("DATABASE_URL="):
|
||||
dsn = line.split("=", 1)[1].strip()
|
||||
break
|
||||
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
|
||||
# Postgres доступен только внутри compose-сети; одноразовые psql-контейнеры
|
||||
# запускаются в этой же сети, поэтому имя хоста postgres резолвится как есть.
|
||||
return dsn
|
||||
|
||||
|
||||
def _psql(dsn_in_container: str, sql: str) -> str:
|
||||
r = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn_in_container, "-tA", "-c", sql,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
return r.stdout + r.stderr
|
||||
|
||||
|
||||
def _scalar(dsn: str, sql: str) -> str:
|
||||
r = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1", "-c", sql,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
|
||||
return r.stdout.strip().splitlines()[-1]
|
||||
|
||||
|
||||
def test_explain(c):
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) истории сообщений на >=10 000 строк.
|
||||
|
||||
Методичка: индекс истории обоснован запросом и EXPLAIN на >=10 000
|
||||
сообщениях. На пустой таблице планировщик выбирает Seq Scan, поэтому
|
||||
тест сам доводит messages до >=10 000 строк и делает ANALYZE.
|
||||
|
||||
Данные сидируются напрямую через psql, без API: тест не зависит от
|
||||
SESSION_TTL_SECONDS и скорости scrypt. Это корректно, потому что
|
||||
EXPLAIN проверяет план SQL-запроса, а не HTTP-слой.
|
||||
"""
|
||||
dsn_in_container = _read_dsn()
|
||||
|
||||
# Смотрим, сколько строк уже есть.
|
||||
have_raw = _psql(dsn_in_container, "SELECT count(*) FROM messages;")
|
||||
have = int(have_raw.strip().splitlines()[-1])
|
||||
print(" в таблице messages сейчас: %d строк" % have)
|
||||
|
||||
target_total = 10_000
|
||||
if have < target_total:
|
||||
need = target_total - have
|
||||
print(" сидируем %d сообщений напрямую через psql ..." % need)
|
||||
|
||||
# Один чат: берём существующий или создаём минимальный.
|
||||
# gen_random_uuid() доступен в PG13+ без расширений.
|
||||
seed_sql = """
|
||||
DO $$
|
||||
DECLARE
|
||||
v_user UUID;
|
||||
v_chat UUID;
|
||||
BEGIN
|
||||
SELECT id INTO v_user FROM users LIMIT 1;
|
||||
IF v_user IS NULL THEN
|
||||
v_user := gen_random_uuid();
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed',
|
||||
'scrypt$16384$8$1$00$00', now());
|
||||
END IF;
|
||||
|
||||
SELECT id INTO v_chat FROM chats LIMIT 1;
|
||||
IF v_chat IS NULL THEN
|
||||
v_chat := gen_random_uuid();
|
||||
INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'Seed chat', now());
|
||||
INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user);
|
||||
END IF;
|
||||
|
||||
-- Доливаем ровно need строк в этот чат.
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), v_chat, v_user,
|
||||
'seed-' || g, now() - (g || ' milliseconds')::interval
|
||||
FROM generate_series(1, %d) AS g;
|
||||
END $$;
|
||||
""" % need
|
||||
r = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn_in_container, "-c", seed_sql,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
seed_out = r.stdout + r.stderr
|
||||
check("ERROR" not in seed_out, "сидирование не удалось:\n" + seed_out)
|
||||
|
||||
# ANALYZE обязателен: без него планировщик не знает реальных размеров.
|
||||
_psql(dsn_in_container, "ANALYZE messages;")
|
||||
_psql(dsn_in_container, "ANALYZE chat_members;")
|
||||
_psql(dsn_in_container, "ANALYZE chats;")
|
||||
print(" сидирование завершено")
|
||||
|
||||
# EXPLAIN по САМОМУ БОЛЬШОМУ чату, чтобы планировщик имел основания
|
||||
# выбрать индекс, а не Seq Scan.
|
||||
biggest = _psql(
|
||||
dsn_in_container,
|
||||
"SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;",
|
||||
).strip().splitlines()[-1]
|
||||
check(biggest and len(biggest) >= 32, "не удалось выбрать самый большой чат: %r" % biggest)
|
||||
|
||||
sql = (
|
||||
"EXPLAIN (ANALYZE, BUFFERS) "
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
"WHERE chat_id = '%s' "
|
||||
"ORDER BY created_at, id LIMIT 51;" % biggest
|
||||
)
|
||||
r = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn_in_container, "-c", sql,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
out = r.stdout + r.stderr
|
||||
print("EXPLAIN (фрагмент):")
|
||||
for line in out.splitlines()[:12]:
|
||||
print(" ", line)
|
||||
check("Index" in out or "Bitmap Index" in out or "idx_messages_chat_created" in out,
|
||||
"EXPLAIN не показал использования индекса:\n" + out)
|
||||
|
||||
|
||||
# ---------- 3. Rollback составной операции create_chat ----------
|
||||
def test_chat_rollback(c):
|
||||
"""Проверяем, что при падении INSERT в chat_members ПОСЛЕ успешного
|
||||
INSERT в chats не остаётся ни строки чата, ни частичного членства.
|
||||
|
||||
Как добиться ошибки именно на вставке участников, а не на предварительной
|
||||
проверке unknown_member? Проверка `SELECT id FROM users WHERE id = ANY(...)`
|
||||
в postgres.py отсекает неизвестных до INSERT. Значит, нужен сбой именно
|
||||
в executemany.
|
||||
|
||||
Решение: запускаем патч-скрипт ВНУТРИ контейнера API через docker exec —
|
||||
там уже установлен psycopg и доступен DATABASE_URL. Скрипт monkey-patch-ит
|
||||
psycopg.Cursor.executemany так, чтобы он бросил исключение, и вызывает
|
||||
storage.create_chat с двумя реально существующими пользователями.
|
||||
Транзакция дойдёт до INSERT в chats, упадёт на executemany и должна
|
||||
откатиться.
|
||||
|
||||
После этого через psql проверяем:
|
||||
- в таблице chats нет строки с этим title;
|
||||
- в таблице chat_members нет строк с этим chat_id;
|
||||
- через API GET /api/v1/chats пользователь не видит этот чат.
|
||||
"""
|
||||
dsn = _read_dsn()
|
||||
|
||||
# Готовим двух валидных пользователей через API, чтобы FK был валиден.
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
alice_username = "rb_a_" + suffix
|
||||
bob_username = "rb_b_" + suffix
|
||||
alice_password = "T3st-" + uuid.uuid4().hex
|
||||
bob_password = "T3st-" + uuid.uuid4().hex
|
||||
alice = c.call("POST", "/api/v1/users", 201,
|
||||
body={"username": alice_username, "display_name": "RB-A",
|
||||
"password": alice_password},
|
||||
schema="User")["json"]
|
||||
bob = c.call("POST", "/api/v1/users", 201,
|
||||
body={"username": bob_username, "display_name": "RB-B",
|
||||
"password": bob_password},
|
||||
schema="User")["json"]
|
||||
|
||||
# Уникальный title, чтобы точно найти строку, если rollback не сработает.
|
||||
marker_title = "rollback-chat-" + uuid.uuid4().hex
|
||||
|
||||
# Скрипт запускается ВНУТРИ контейнера API: там есть psycopg и DATABASE_URL.
|
||||
# Патчит executemany так, чтобы он упал после успешного INSERT в chats.
|
||||
patch_script = r'''
|
||||
import os, sys, psycopg
|
||||
from uuid import UUID
|
||||
|
||||
_orig = psycopg.Cursor.executemany
|
||||
def boom(self, *args, **kwargs):
|
||||
raise RuntimeError("patched_executemany_failure")
|
||||
psycopg.Cursor.executemany = boom
|
||||
|
||||
sys.path.insert(0, "/app/src")
|
||||
from app.storage.postgres import storage
|
||||
|
||||
try:
|
||||
storage.create_chat(os.environ["RB_TITLE"],
|
||||
[UUID(os.environ["RB_ALICE"]), UUID(os.environ["RB_BOB"])])
|
||||
except RuntimeError as exc:
|
||||
if "patched_executemany_failure" in str(exc):
|
||||
print("PATCHED_FAILURE_OK"); sys.exit(0)
|
||||
raise
|
||||
except Exception as exc:
|
||||
print("UNEXPECTED:%s:%s" % (type(exc).__name__, exc)); sys.exit(2)
|
||||
print("NO_FAILURE"); sys.exit(3)
|
||||
'''
|
||||
|
||||
proc = subprocess.run(
|
||||
["docker", "exec",
|
||||
"-e", "RB_TITLE=" + marker_title,
|
||||
"-e", "RB_ALICE=" + alice["id"],
|
||||
"-e", "RB_BOB=" + bob["id"],
|
||||
API_CONTAINER,
|
||||
"python", "-c", patch_script],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
out = (proc.stdout + proc.stderr).strip()
|
||||
print(" rollback-процесс: exit=%d, out=%r" % (proc.returncode, out[:200]))
|
||||
check("PATCHED_FAILURE_OK" in out,
|
||||
"не удалось вызвать ошибку именно на executemany: %s" % out)
|
||||
|
||||
# Проверяем в БД: ни chats, ни chat_members с этим маркером не осталось.
|
||||
chats_count = _scalar(
|
||||
dsn,
|
||||
"SELECT count(*) FROM chats WHERE title = '%s';" % marker_title,
|
||||
)
|
||||
check(int(chats_count) == 0,
|
||||
"rollback не сработал: строка chats с title=%s осталась (%s)" % (marker_title, chats_count))
|
||||
|
||||
members_count = _scalar(
|
||||
dsn,
|
||||
"SELECT count(*) FROM chat_members cm "
|
||||
"JOIN chats c ON c.id = cm.chat_id "
|
||||
"WHERE c.title = '%s';" % marker_title,
|
||||
)
|
||||
check(int(members_count) == 0,
|
||||
"rollback не сработал: остались строки chat_members (%s)" % members_count)
|
||||
|
||||
# Дополнительно: у пользователя нет чата с таким title через API.
|
||||
cred = base64.b64encode((alice_username + ":" + alice_password).encode()).decode()
|
||||
session = c.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
schema="Session")["json"]
|
||||
listed = c.call("GET", "/api/v1/chats?limit=100",
|
||||
headers={"Authorization": "Bearer " + session["token"]},
|
||||
schema="ChatPage")["json"]
|
||||
check(all(item["title"] != marker_title for item in listed["items"]),
|
||||
"чат с маркером rollback виден в списке чатов пользователя")
|
||||
|
||||
|
||||
# ---------- 4. Недоступность Redis (с действующим токеном) ----------
|
||||
def _compose(action, service="redis"):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", action, service],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _wait_ready(c, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API не вернулся в ready за %d секунд после восстановления зависимости" % timeout)
|
||||
|
||||
|
||||
def test_redis_unavailable(c):
|
||||
"""Останавливаем Redis: /health/ready → 503, действующий токен не работает.
|
||||
|
||||
Важно: используем ЗАРАНЕЕ полученную рабочую сессию, а не заведомо
|
||||
неправильный Bearer. Иначе 401 мог бы получиться и без обращения к Redis.
|
||||
После восстановления Redis прежняя сессия снова должна работать.
|
||||
"""
|
||||
print(" (проверка недоступности Redis: docker compose stop redis на ~6 секунд)")
|
||||
|
||||
# Готовим рабочую сессию ДО остановки Redis.
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
username = "redis_" + suffix
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
c.call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": "Redis", "password": password},
|
||||
schema="User")
|
||||
cred = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = c.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
schema="Session")["json"]
|
||||
token = session["token"]
|
||||
|
||||
# Контроль: до остановки Redis токен рабочий.
|
||||
me = c.call("GET", "/api/v1/users/me", 200,
|
||||
headers={"Authorization": "Bearer " + token},
|
||||
schema="User")["json"]
|
||||
check(me["username"] == username, "токен должен принадлежать созданному пользователю")
|
||||
|
||||
stopped = _compose("stop", "redis")
|
||||
check(stopped.returncode == 0,
|
||||
"docker compose stop redis не сработал:\n" + (stopped.stdout + stopped.stderr))
|
||||
|
||||
try:
|
||||
time.sleep(3)
|
||||
# /health/ready должен стать 503.
|
||||
ready = c.call("GET", "/health/ready", 503)
|
||||
check(ready["status"] == 503, "readiness must be 503 without Redis")
|
||||
|
||||
# Действующий токен при мёртвом Redis не должен давать доступ:
|
||||
# сервер не может проверить сессию и обязан ответить 503.
|
||||
r = c.call("GET", "/api/v1/users/me", (401, 503),
|
||||
headers={"Authorization": "Bearer " + token})
|
||||
check(r["status"] == 503,
|
||||
"запрос с действующим токеном при мёртвом Redis должен быть 503, получено %s" % r["status"])
|
||||
finally:
|
||||
# Только восстановление. Никаких return/break/continue внутри finally.
|
||||
started = _compose("start", "redis")
|
||||
check(started.returncode == 0,
|
||||
"docker compose start redis не сработал:\n" + (started.stdout + started.stderr))
|
||||
|
||||
# Ждём готовности — УЖЕ ВНЕ finally.
|
||||
_wait_ready(c, timeout=60)
|
||||
|
||||
# Прежний токен снова должен работать: сессия пережила рестарт Redis (AOF).
|
||||
me2 = c.call("GET", "/api/v1/users/me", 200,
|
||||
headers={"Authorization": "Bearer " + token},
|
||||
schema="User")["json"]
|
||||
check(me2["id"] == me["id"], "прежняя сессия должна снова работать после восстановления Redis")
|
||||
|
||||
|
||||
# ---------- 5. Недоступность PostgreSQL ----------
|
||||
def test_postgres_unavailable(c):
|
||||
"""Останавливаем PostgreSQL: /health/ready → 503, действующий токен → 503.
|
||||
|
||||
Проверяем:
|
||||
- /health/ready возвращает 503;
|
||||
- запрос с действующим токеном возвращает 503 в пределах таймаута;
|
||||
- анонимный запрос не получает доступ;
|
||||
- после запуска БД прежний токен работает, ранее записанные данные сохранены.
|
||||
Восстановление БД выполняется в finally, чтобы не оставлять стенд остановленным.
|
||||
"""
|
||||
print(" (проверка недоступности PostgreSQL: docker compose stop postgres на ~6 секунд)")
|
||||
|
||||
# Готовим рабочую сессию и сообщение ДО остановки БД.
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
marker = "pgdown-" + uuid.uuid4().hex
|
||||
sent = c.message(alice, chat, marker)["json"]
|
||||
|
||||
# Контроль: до остановки всё работает.
|
||||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||||
|
||||
stopped = _compose("stop", "postgres")
|
||||
check(stopped.returncode == 0,
|
||||
"docker compose stop postgres не сработал:\n" + (stopped.stdout + stopped.stderr))
|
||||
|
||||
try:
|
||||
time.sleep(3)
|
||||
# /health/ready должен стать 503.
|
||||
ready = c.call("GET", "/health/ready", 503)
|
||||
check(ready["status"] == 503, "readiness must be 503 without PostgreSQL")
|
||||
|
||||
# Запрос с действующим токеном: 503 в пределах таймаута.
|
||||
t0 = time.monotonic()
|
||||
r = c.call("GET", "/api/v1/users/me", 503, actor=alice)
|
||||
elapsed = time.monotonic() - t0
|
||||
check(r["status"] == 503,
|
||||
"запрос с действующим токеном при мёртвой БД должен быть 503")
|
||||
# statement_timeout=5000 + пул timeout=5.0: укладываемся в разумный предел.
|
||||
check(elapsed < 30, "ответ при мёртвой БД занял %.1f c, ожидался в пределах таймаута" % elapsed)
|
||||
|
||||
# Анонимный запрос не получает доступ.
|
||||
anon = c.call("GET", "/api/v1/users/me", (401, 503))
|
||||
check(anon["status"] in (401, 503),
|
||||
"анонимный запрос при мёртвой БД должен быть 401 или 503, получено %s" % anon["status"])
|
||||
finally:
|
||||
started = _compose("start", "postgres")
|
||||
check(started.returncode == 0,
|
||||
"docker compose start postgres не сработал:\n" + (started.stdout + started.stderr))
|
||||
|
||||
# Ждём готовности — УЖЕ ВНЕ finally.
|
||||
_wait_ready(c, timeout=60)
|
||||
|
||||
# Прежний токен работает, ранее записанное сообщение сохранено.
|
||||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||||
messages = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=bob, schema="MessagePage")["json"]["items"]
|
||||
check(any(m["id"] == sent["id"] and m["text"] == marker for m in messages),
|
||||
"ранее записанное сообщение не найдено после восстановления PostgreSQL")
|
||||
|
||||
|
||||
# ---------- 6. Expiry TTL ----------
|
||||
def test_short_ttl_expiry(c):
|
||||
"""TTL=5. Проверяем полный цикл: токен валиден -> истекает -> 401.
|
||||
|
||||
Идёт ПОСЛЕДНИМ: при коротком TTL любая долгая подготовка съедает
|
||||
окно жизни сессии, поэтому соседние тесты работают раньше.
|
||||
"""
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
username = "ttl_" + suffix
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
user = c.call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": "TTL", "password": password},
|
||||
schema="User")["json"]
|
||||
cred = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = c.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
schema="Session")["json"]
|
||||
token = session["token"]
|
||||
|
||||
# Контроль: сервер выдал короткий TTL, иначе тест бессмыслен.
|
||||
expires_at = dt.datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00"))
|
||||
ttl_actual = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
check(3.0 <= ttl_actual <= 10,
|
||||
"SESSION_TTL_SECONDS не короткий (получено %.1f c). "
|
||||
"Поставьте, пожалуйста, SESSION_TTL_SECONDS=5 в .env и перезапустите API (команда: docker compose --env-file .env up -d api)." % ttl_actual)
|
||||
|
||||
# До истечения токен должен работать.
|
||||
me = c.call("GET", "/api/v1/users/me", 200,
|
||||
headers={"Authorization": "Bearer " + token},
|
||||
schema="User")["json"]
|
||||
check(me["id"] == user["id"], "token must identify its owner before expiry")
|
||||
|
||||
# Ждём гарантированного истечения TTL + запас.
|
||||
remaining = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
time.sleep(max(remaining, 0.0) + 1.0)
|
||||
|
||||
# Истёкший токен — 401.
|
||||
c.call("GET", "/api/v1/users/me", 401,
|
||||
headers={"Authorization": "Bearer " + token})
|
||||
|
||||
# Второй вход тем же пользователем даёт рабочий токен: истёк именно
|
||||
# первый токен, а не пользователь/хранилище целиком.
|
||||
session2 = c.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
schema="Session")["json"]
|
||||
c.call("GET", "/api/v1/users/me", 200,
|
||||
headers={"Authorization": "Bearer " + session2["token"]},
|
||||
schema="User")
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
def main():
|
||||
args = arguments("Lab2 level 4 checks", configure=_configure)
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
|
||||
# Порядок и режимы:
|
||||
# --only-expiry — ТОЛЬКО expiry-тест (запускать при SESSION_TTL_SECONDS=5)
|
||||
# --skip-expiry — все тяжёлые тесты, без expiry (при SESSION_TTL_SECONDS=3600)
|
||||
# без флагов — сначала тяжёлые, expiry последним (при SESSION_TTL_SECONDS=5
|
||||
# это сработает только если тяжёлые тесты успевают до истечения TTL,
|
||||
# поэтому на практике используйте --only-expiry/--skip-expiry)
|
||||
run_expiry = not args.skip_expiry
|
||||
run_others = not args.only_expiry
|
||||
|
||||
if run_others:
|
||||
print("=== test_concurrent_registration ===")
|
||||
test_concurrent_registration(c)
|
||||
print("PASS: concurrent registration produces exactly one user")
|
||||
|
||||
print("=== test_chat_rollback ===")
|
||||
test_chat_rollback(c)
|
||||
print("PASS: create_chat rollback leaves no chat and no partial membership")
|
||||
|
||||
print("=== test_explain ===")
|
||||
test_explain(c)
|
||||
print("PASS: EXPLAIN uses idx_messages_chat_created")
|
||||
|
||||
print("=== test_redis_unavailable ===")
|
||||
test_redis_unavailable(c)
|
||||
print("PASS: readiness 503 без Redis, действующий токен отвергнут, затем восстановление")
|
||||
|
||||
print("=== test_postgres_unavailable ===")
|
||||
test_postgres_unavailable(c)
|
||||
print("PASS: readiness 503 без PostgreSQL, данные и сессия сохранены")
|
||||
|
||||
if run_expiry:
|
||||
print("=== test_short_ttl_expiry ===")
|
||||
test_short_ttl_expiry(c)
|
||||
print("PASS: session valid before TTL, expires after TTL, 401 on expired token")
|
||||
|
||||
print("PASS: all requested lab2 level-4 checks passed")
|
||||
|
||||
|
||||
def _configure(p):
|
||||
p.add_argument("--only-expiry", action="store_true",
|
||||
help="запустить только expiry-тест (SESSION_TTL_SECONDS=5)")
|
||||
p.add_argument("--skip-expiry", action="store_true",
|
||||
help="запустить всё, кроме expiry (SESSION_TTL_SECONDS=3600)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,465 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO.
|
||||
|
||||
Шаги:
|
||||
1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql).
|
||||
2. Сравниваем 3 варианта индексов на большом и маленьком чате:
|
||||
- baseline (нет индекса) -> Seq Scan
|
||||
- (chat_id) -> Bitmap/Index + Sort
|
||||
- (chat_id, created_at, id) -> Index Scan без Sort (текущий)
|
||||
Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read.
|
||||
3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки,
|
||||
pg_indexes_size, pg_total_relation_size.
|
||||
4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine,
|
||||
сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений.
|
||||
5. Границы потери данных: во время pg_dump через API пишем новые сообщения,
|
||||
после restore проверяем, что их нет.
|
||||
|
||||
Все psql-вызовы идут через docker run postgres:18-alpine в compose-сети проекта
|
||||
(--network mtuci2026lab2_default), DSN указывает на хост postgres.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import base64
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
|
||||
SEED_SQL = ROOT / "student-tests" / "track2_seed.sql"
|
||||
|
||||
# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name).
|
||||
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default")
|
||||
|
||||
|
||||
# ---------- helpers ----------
|
||||
def _read_dsn() -> str:
|
||||
dsn = os.environ.get("DATABASE_URL")
|
||||
if not dsn:
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.is_file():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("DATABASE_URL="):
|
||||
dsn = line.split("=", 1)[1].strip()
|
||||
break
|
||||
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
|
||||
# Postgres доступен только внутри compose-сети; одноразовые psql/pg_dump
|
||||
# контейнеры запускаются в этой же сети, имя хоста postgres резолвится как есть.
|
||||
return dsn
|
||||
|
||||
|
||||
def _psql(dsn: str, sql: str, extra=None):
|
||||
cmd = [
|
||||
"docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1",
|
||||
]
|
||||
if extra:
|
||||
cmd += extra
|
||||
cmd += ["-c", sql]
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=300)
|
||||
|
||||
|
||||
def _psql_file(dsn: str, path: Path):
|
||||
cmd = [
|
||||
"docker", "run", "--rm", "-i",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-v", f"{path}:/seed.sql:ro",
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql",
|
||||
]
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=600)
|
||||
|
||||
|
||||
def _scalar(dsn: str, sql: str):
|
||||
r = _psql(dsn, sql)
|
||||
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
|
||||
return r.stdout.strip().splitlines()[-1]
|
||||
|
||||
|
||||
# ---------- 1. seed ----------
|
||||
def step_seed(dsn: str):
|
||||
print("=== 1. seed ===")
|
||||
before = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
||||
print(" messages before:", before)
|
||||
r = _psql_file(dsn, SEED_SQL)
|
||||
check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr)
|
||||
after = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
||||
print(" messages after:", after)
|
||||
check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after)
|
||||
dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;")
|
||||
print(" распределение по чатам:\n" + dist.stdout)
|
||||
return after
|
||||
|
||||
|
||||
# ---------- 2. explain ----------
|
||||
def _big_and_small(dsn):
|
||||
big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;")
|
||||
small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;")
|
||||
return big, small
|
||||
|
||||
|
||||
def _chat_size(dsn: str, chat_id: str) -> int:
|
||||
return int(_scalar(dsn, "SELECT count(*) FROM messages WHERE chat_id = '%s';" % chat_id))
|
||||
|
||||
|
||||
def _explain(dsn: str, chat_id: str, label: str):
|
||||
# Берём средний created_at, чтобы keyset реально резал историю
|
||||
mid = _scalar(dsn,
|
||||
"SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id))
|
||||
sql = (
|
||||
"EXPLAIN (ANALYZE, BUFFERS) "
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
"WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') "
|
||||
"ORDER BY created_at, id LIMIT 51;" % (chat_id, mid)
|
||||
)
|
||||
r = _psql(dsn, sql)
|
||||
check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr)
|
||||
out = r.stdout
|
||||
print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8]))
|
||||
for line in out.splitlines():
|
||||
print(" ", line)
|
||||
return out
|
||||
|
||||
|
||||
def _drop_indexes(dsn):
|
||||
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;")
|
||||
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
|
||||
|
||||
|
||||
def _format_section(title: str, chat_size: int, plan: str, note: str = "") -> str:
|
||||
"""Собирает один раздел отчёта в формате образца."""
|
||||
head = "## %s\n\n" % title
|
||||
if note:
|
||||
head += note.rstrip() + "\n\n"
|
||||
return head + "```\n" + plan.rstrip() + "\n```\n"
|
||||
|
||||
|
||||
def step_explain(dsn: str, total_messages: int):
|
||||
print("=== 2. EXPLAIN 3 варианта ===")
|
||||
big, small = _big_and_small(dsn)
|
||||
big_size = _chat_size(dsn, big)
|
||||
small_size = _chat_size(dsn, small)
|
||||
|
||||
# (1) без индекса
|
||||
_drop_indexes(dsn)
|
||||
_psql(dsn, "ANALYZE messages;")
|
||||
no_index_big = _explain(dsn, big, "no_index/big")
|
||||
no_index_small = _explain(dsn, small, "no_index/small")
|
||||
|
||||
# (2) только (chat_id)
|
||||
_psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);")
|
||||
_psql(dsn, "ANALYZE messages;")
|
||||
chat_only_big = _explain(dsn, big, "chat_only/big")
|
||||
chat_only_small = _explain(dsn, small, "chat_only/small")
|
||||
|
||||
# (3) полный рабочий
|
||||
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
|
||||
_psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);")
|
||||
_psql(dsn, "ANALYZE messages;")
|
||||
full_big = _explain(dsn, big, "full/big")
|
||||
full_small = _explain(dsn, small, "full/small")
|
||||
|
||||
# Заголовок отчёта — как в образце
|
||||
parts = []
|
||||
parts.append(
|
||||
"# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса\n\n"
|
||||
"Данные: %d сообщений, неравномерные чаты. "
|
||||
"Запрос: `SELECT ... FROM messages WHERE chat_id = <id> AND (created_at, id) > (<середина>, …) "
|
||||
"ORDER BY created_at, id LIMIT 51`.\n"
|
||||
"Большой чат ~%d, маленький чат ~%d.\n" % (total_messages, big_size, small_size)
|
||||
)
|
||||
|
||||
parts.append(_format_section(
|
||||
"1. Без индекса, большой чат", big_size, no_index_big,
|
||||
"`Parallel Seq Scan` по всей таблице, затем `Sort`."))
|
||||
parts.append(_format_section(
|
||||
"2. Без индекса, маленький чат", small_size, no_index_small,
|
||||
"Тот же `Parallel Seq Scan` по всей таблице."))
|
||||
parts.append(_format_section(
|
||||
"3. Индекс (chat_id), большой чат", big_size, chat_only_big,
|
||||
"`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`."))
|
||||
parts.append(_format_section(
|
||||
"4. Индекс (chat_id), маленький чат", small_size, chat_only_small,
|
||||
"`Index Scan using idx_messages_chat_only` + `Sort`."))
|
||||
parts.append(_format_section(
|
||||
"5. Полный индекс (chat_id, created_at, id), большой чат", big_size, full_big,
|
||||
"`Index Scan using idx_messages_chat_created`, **без `Sort`**."))
|
||||
parts.append(_format_section(
|
||||
"6. Полный индекс (chat_id, created_at, id), маленький чат", small_size, full_small,
|
||||
"`Index Scan` без `Sort`."))
|
||||
|
||||
(EVIDENCE / "lab2_track2_explain.md").write_text(
|
||||
"\n".join(parts), encoding="utf-8")
|
||||
|
||||
return {"big_chat": big, "small_chat": small}
|
||||
|
||||
|
||||
# ---------- 3. write cost ----------
|
||||
def _index_sizes(dsn: str):
|
||||
out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));")
|
||||
idx, total = out.split("|")
|
||||
return idx.strip(), total.strip()
|
||||
|
||||
|
||||
def step_write_cost(dsn: str):
|
||||
print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===")
|
||||
seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;")
|
||||
seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;")
|
||||
|
||||
results = []
|
||||
for name, setup in (
|
||||
("no_index", lambda: _drop_indexes(dsn)),
|
||||
("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"),
|
||||
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))),
|
||||
("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"),
|
||||
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))),
|
||||
):
|
||||
setup()
|
||||
# мерить именно вставку 10k
|
||||
t0 = time.monotonic()
|
||||
r = _psql(dsn,
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
||||
"SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() "
|
||||
"FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name))
|
||||
dt = time.monotonic() - t0
|
||||
check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr)
|
||||
idx, total = _index_sizes(dsn)
|
||||
results.append({"variant": name, "insert_10000_s": round(dt, 3),
|
||||
"indexes_size": idx, "total_size": total})
|
||||
print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total))
|
||||
|
||||
(EVIDENCE / "lab2_track2_write_cost.json").write_text(
|
||||
json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return results
|
||||
|
||||
|
||||
# ---------- 4. backup/restore ----------
|
||||
def step_backup_restore(dsn: str):
|
||||
print("=== 4. backup -> restore -> сверка ===")
|
||||
out_file = EVIDENCE / "lab2_track2_dump.dump"
|
||||
if out_file.exists():
|
||||
out_file.unlink()
|
||||
|
||||
# pg_dump через docker run (использует образ PG 18), в compose-сети
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-v", f"{EVIDENCE}:/out",
|
||||
"postgres:18-alpine",
|
||||
"pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
dump_time = time.monotonic() - t0
|
||||
check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr)
|
||||
size = out_file.stat().st_size
|
||||
print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time))
|
||||
|
||||
# новый контейнер как отдельное хранилище — в той же compose-сети,
|
||||
# порт наружу не публикуем
|
||||
restore_container = "mtuci-pg-restore"
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
subprocess.run(
|
||||
["docker", "run", "-d", "--name", restore_container,
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
|
||||
"-e", "POSTGRES_DB=high_load_messenger",
|
||||
"postgres:18-alpine"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
# ждём готовности
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
|
||||
capture_output=True, text=True)
|
||||
if c.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("restore container не поднялся")
|
||||
|
||||
# pg_restore -c (clean) на пустую БД; restore-контейнер доступен по имени
|
||||
# в compose-сети
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-v", f"{EVIDENCE}:/in",
|
||||
"postgres:18-alpine",
|
||||
"pg_restore", "-d", "postgresql://postgres:restore@mtuci-pg-restore:5432/high_load_messenger",
|
||||
"--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
restore_time = time.monotonic() - t0
|
||||
print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time))
|
||||
check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr)
|
||||
|
||||
# Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера
|
||||
def q_restore(sql):
|
||||
c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
|
||||
"-d", "high_load_messenger", "-tA", "-c", sql],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr)
|
||||
return c.stdout.strip().splitlines()[-1]
|
||||
|
||||
orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
||||
rest_total = int(q_restore("SELECT count(*) FROM messages;"))
|
||||
check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total))
|
||||
|
||||
orig_dist = _scalar(dsn,
|
||||
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
|
||||
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
|
||||
rest_dist = q_restore(
|
||||
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
|
||||
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
|
||||
check(orig_dist == rest_dist, "распределение по чатам не совпадает")
|
||||
|
||||
# 100 случайных сообщений
|
||||
ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;")
|
||||
ids = ids_raw.split(",")
|
||||
check(len(ids) == 100, "не удалось выбрать 100 id")
|
||||
ids_list = ",".join("'%s'" % i for i in ids)
|
||||
orig_rows = _scalar(dsn,
|
||||
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
|
||||
rest_rows = q_restore(
|
||||
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
|
||||
check(orig_rows == rest_rows, "100 случайных сообщений различаются")
|
||||
|
||||
print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают")
|
||||
print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024))
|
||||
|
||||
(EVIDENCE / "lab2_track2_restore.txt").write_text(
|
||||
"Track 2 restore\n"
|
||||
"dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n"
|
||||
"count_original=%d\ncount_restored=%d\n"
|
||||
"dist_match=true\nrandom100_match=true\n"
|
||||
% (dump_time, size, restore_time, orig_total, rest_total),
|
||||
encoding="utf-8")
|
||||
|
||||
# чистим restore-контейнер (volume анонимный)
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3),
|
||||
"dump_bytes": size, "count": orig_total}
|
||||
|
||||
|
||||
# ---------- 5. RPO ----------
|
||||
def step_rpo(dsn: str, base_url: str, ca_file):
|
||||
print("=== 5. RPO: запись во время pg_dump ===")
|
||||
c = Client(base_url, ca_file)
|
||||
|
||||
# Данные и сессии готовим заранее, до дампа.
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
# Контроль: сессия должна быть валидна, иначе 401 — это не RPO.
|
||||
me = c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||||
check(me["status"] == 200, "сессия должна быть валидна перед RPO-тестом")
|
||||
|
||||
marker = "rpo-" + uuid.uuid4().hex
|
||||
# Запускаем pg_dump
|
||||
out_file = EVIDENCE / "lab2_track2_rpo.dump"
|
||||
if out_file.exists():
|
||||
out_file.unlink()
|
||||
|
||||
# pg_dump: тот же приём, что в шаге 4, в compose-сети.
|
||||
proc = subprocess.Popen(
|
||||
["docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-v", f"{EVIDENCE}:/out",
|
||||
"postgres:18-alpine",
|
||||
"pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
time.sleep(1.0) # снапшот уже снят
|
||||
|
||||
# Свежий логин прямо перед записью: не зависим от TTL старой сессии.
|
||||
cred = base64.b64encode(
|
||||
(alice["registration"]["username"] + ":" +
|
||||
alice["registration"]["password"]).encode()
|
||||
).decode()
|
||||
fresh = c.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
schema="Session")["json"]
|
||||
alice_for_write = {"id": alice["id"], "user": alice["user"], "token": fresh["token"]}
|
||||
|
||||
written = c.message(alice_for_write, chat, marker)["json"]
|
||||
code = proc.wait(timeout=300)
|
||||
check(code == 0, "pg_dump для RPO упал")
|
||||
|
||||
# Временный restore-контейнер: в compose-сети, без публикации портов.
|
||||
restore_container = "mtuci-pg-rpo"
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
subprocess.run(
|
||||
["docker", "run", "-d", "--name", restore_container,
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
|
||||
"-e", "POSTGRES_DB=high_load_messenger",
|
||||
"postgres:18-alpine"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
|
||||
capture_output=True, text=True)
|
||||
if c2.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("restore container для RPO не поднялся")
|
||||
|
||||
# pg_restore идёт в restore-контейнер по имени в compose-сети.
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm",
|
||||
"--network", COMPOSE_NETWORK,
|
||||
"-v", f"{EVIDENCE}:/in",
|
||||
"postgres:18-alpine",
|
||||
"pg_restore", "-d",
|
||||
"postgresql://postgres:restore@mtuci-pg-rpo:5432/high_load_messenger",
|
||||
"--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr)
|
||||
|
||||
c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
|
||||
"-d", "high_load_messenger", "-tA", "-c",
|
||||
"SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]],
|
||||
capture_output=True, text=True)
|
||||
present = int(c3.stdout.strip().splitlines()[-1])
|
||||
check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)")
|
||||
print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8])
|
||||
|
||||
(EVIDENCE / "lab2_track2_rpo.txt").write_text(
|
||||
"Track 2 RPO\n"
|
||||
"marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n"
|
||||
% (written["id"], present),
|
||||
encoding="utf-8")
|
||||
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
def main():
|
||||
args = arguments("Track 2: plan, write cost, backup/restore, RPO")
|
||||
dsn = _read_dsn()
|
||||
total = step_seed(dsn)
|
||||
step_explain(dsn, total)
|
||||
step_write_cost(dsn)
|
||||
step_backup_restore(dsn)
|
||||
step_rpo(dsn, args.base_url, args.ca_file)
|
||||
print("PASS: track 2 (explain + write cost + backup/restore + RPO)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Track 2: сравнение планов и стоимости записи.
|
||||
-- Все три варианта прогоняются на одном большом чате (60k) и одном маленьком (~1400).
|
||||
|
||||
-- 0. Определяем два чата
|
||||
\set ON_ERROR_STOP on
|
||||
|
||||
SELECT id AS big_chat_id
|
||||
FROM messages GROUP BY id_chat GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;
|
||||
|
||||
SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;
|
||||
@@ -0,0 +1,81 @@
|
||||
-- Track 2: сидирование >=100 000 сообщений с неравномерными чатами.
|
||||
-- Запускать через: docker run --rm ... psql "$DATABASE_URL" -f track2_seed.sql
|
||||
-- Идемпотентно по количеству: если сообщений уже >= 100 000, ничего не делает.
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
v_user UUID;
|
||||
v_total INT;
|
||||
v_have INT;
|
||||
v_need INT;
|
||||
v_chat UUID;
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_have FROM messages;
|
||||
v_total := 100000;
|
||||
SELECT count(*) INTO v_have FROM messages;
|
||||
IF v_have < 100000 THEN
|
||||
-- меньше 100к: догоняем до 100к
|
||||
v_need := 100000 - v_have; -- сколько добавлять за прогон
|
||||
ELSE
|
||||
v_need := 50000; -- добавляем фиксированные 50к за прогон
|
||||
END IF;
|
||||
|
||||
-- Гарантируем хотя бы одного пользователя
|
||||
SELECT id INTO v_user FROM users LIMIT 1;
|
||||
IF v_user IS NULL THEN
|
||||
v_user := gen_random_uuid();
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed',
|
||||
'scrypt$16384$8$1$00$00', now());
|
||||
END IF;
|
||||
|
||||
-- Создаём 10 чатов, если их меньше (нужны только для распределения)
|
||||
-- Неравномерность: 60k / 20k / 10k / по ~1400 на остальные 7.
|
||||
-- Считаем веса, доливаем только v_need строк.
|
||||
CREATE TEMP TABLE _plan(chat_no INT, weight INT) ON COMMIT DROP;
|
||||
INSERT INTO _plan VALUES (1, 60), (2, 20), (3, 10),
|
||||
(4, 2), (5, 2), (6, 1), (7, 1),
|
||||
(8, 1), (9, 1), (10, 1);
|
||||
|
||||
FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP
|
||||
-- Находим/создаём chat_id для этого номера: используем title 'seed-chat-<no>'
|
||||
SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1;
|
||||
IF v_chat IS NULL THEN
|
||||
v_chat := gen_random_uuid();
|
||||
INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'seed-chat-' || rec.chat_no, now());
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = v_chat AND user_id = v_user) THEN
|
||||
INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user);
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- Распределяем v_need по весам (целочисленно), последний чат добирает остаток.
|
||||
DECLARE
|
||||
v_sum INT;
|
||||
v_acc INT := 0;
|
||||
v_alloc INT;
|
||||
BEGIN
|
||||
SELECT sum(weight) INTO v_sum FROM _plan;
|
||||
FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP
|
||||
IF rec.chat_no = 10 THEN
|
||||
v_alloc := v_need - v_acc; -- остаток
|
||||
ELSE
|
||||
v_alloc := (v_need::numeric * rec.weight / v_sum)::int;
|
||||
v_acc := v_acc + v_alloc;
|
||||
END IF;
|
||||
|
||||
SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), v_chat, v_user,
|
||||
'seed-' || rec.chat_no || '-' || g,
|
||||
now() - (g || ' milliseconds')::interval
|
||||
FROM generate_series(1, v_alloc) AS g;
|
||||
END LOOP;
|
||||
END;
|
||||
END $$;
|
||||
|
||||
ANALYZE messages;
|
||||
ANALYZE chat_members;
|
||||
ANALYZE chats;
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4 lab3: два X-Instance-Id, идемпотентность под нагрузкой, отказ одной реплики.
|
||||
|
||||
Запуск:
|
||||
python student-tests/lab3_test_level4.py --base-url http://localhost:8080
|
||||
|
||||
Требуется рабочий стенд из compose.yaml (LB + 2 реплики + Postgres + Redis).
|
||||
"""
|
||||
import base64
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
|
||||
# ---------- 1. Два X-Instance-Id через LB ----------
|
||||
def test_two_instances(c):
|
||||
"""Один пользователь, серия запросов — видим >=2 разных X-Instance-Id.
|
||||
|
||||
Nginx с keepalive к upstream прибивает клиента к одной реплике на всё
|
||||
время соединения. Поэтому каждый проб-запрос шлём с Connection: close,
|
||||
чтобы LB открывал новое upstream-соединение и честно распределял.
|
||||
"""
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
seen = set()
|
||||
for _ in range(60):
|
||||
r = c.call("GET", "/api/v1/users/me", actor=alice, schema="User",
|
||||
headers={"Connection": "close"})
|
||||
seen.add(r["headers"]["X-Instance-Id"])
|
||||
if len(seen) >= 2:
|
||||
break
|
||||
check(len(seen) >= 2, "LB не маршрутизирует на разные API-реплики: %s" % seen)
|
||||
print(" observed instances:", sorted(seen))
|
||||
|
||||
|
||||
# ---------- 2. Идемпотентность: один ключ на двух репликах ----------
|
||||
def test_idempotency_across_replicas(c):
|
||||
"""Одновременный POST с одним Idempotency-Key через LB создаёт ровно одно сообщение.
|
||||
|
||||
Для этого параллельно шлём 10 запросов с одним ключом из 10 потоков.
|
||||
LB распределит их по разным репликам; благодаря общей таблице
|
||||
idempotency_keys и UNIQUE-индексу, INSERT выигрывает ровно один.
|
||||
"""
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
key = "idem-" + uuid.uuid4().hex
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
body = {"text": "idem-concurrent"}
|
||||
lock = threading.Lock()
|
||||
results = []
|
||||
|
||||
def one(_):
|
||||
local = Client(c.base, timeout=10)
|
||||
try:
|
||||
r = local.call(
|
||||
"POST", path, (201, 409),
|
||||
actor=alice, body=body,
|
||||
headers={"Idempotency-Key": key},
|
||||
schema="Message",
|
||||
)
|
||||
with lock:
|
||||
results.append((r["status"], r["headers"].get("X-Instance-Id")))
|
||||
except Failure as exc:
|
||||
with lock:
|
||||
results.append(("failure:%s" % exc, None))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
list(pool.map(one, range(10)))
|
||||
|
||||
ok = [r for r in results if r[0] == 201]
|
||||
conflicts = [r for r in results if r[0] == 409]
|
||||
bad = [r for r in results if r[0] not in (201, 409)]
|
||||
check(not bad, "unexpected idempotency results: %s" % results)
|
||||
check(len(ok) >= 1, "no 201 at all")
|
||||
check(len(ok) + len(conflicts) == 10, "wrong count")
|
||||
|
||||
# Проверяем в БД: ровно одно сообщение с этим текстом в этом чате.
|
||||
# Через API: читаем всю историю и считаем.
|
||||
from urllib.parse import quote
|
||||
items = []
|
||||
cursor = None
|
||||
for _ in range(50):
|
||||
q = path + "?limit=100"
|
||||
if cursor:
|
||||
q += "&cursor=" + quote(cursor, safe="")
|
||||
page = c.call("GET", q, actor=bob, schema="MessagePage")["json"]
|
||||
items.extend(m for m in page["items"] if m["text"] == "idem-concurrent")
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
check(len(items) == 1, "idempotency created %d messages, expected 1" % len(items))
|
||||
|
||||
# X-Instance-Id: запросы шли на разные реплики (проверяем, что 201 пришёл
|
||||
# хотя бы с одной, а 409 — с другой; это ожидание не обязательное, но
|
||||
# полезный сигнал, что LB реально распределяет).
|
||||
ok_instances = {r[1] for r in ok}
|
||||
conflict_instances = {r[1] for r in conflicts}
|
||||
print(" 201 instances:", ok_instances, " 409 instances:", conflict_instances)
|
||||
print(" PASS: idempotency OK, ровно одно сообщение создано")
|
||||
|
||||
|
||||
# ---------- 3. Отказ одной реплики ----------
|
||||
def test_stop_one_replica(c):
|
||||
"""Останавливаем одну реплику; вторая продолжает обслуживать запросы.
|
||||
|
||||
Восстановление делаем явно через `docker start` в finally, а ожидание
|
||||
готовности — ПОСЛЕ try/finally, чтобы не использовать return в finally.
|
||||
"""
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
|
||||
# Список контейнеров сервиса api (compose project name неизвестен,
|
||||
# ищем по image/label).
|
||||
ps = subprocess.run(
|
||||
["docker", "ps", "--filter", "label=com.docker.compose.service=api",
|
||||
"--format", "{{.ID}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
ids = [x.strip() for x in ps.stdout.splitlines() if x.strip()]
|
||||
check(len(ids) >= 2, "нужно >=2 контейнера api, найдено %d" % len(ids))
|
||||
|
||||
target = ids[0]
|
||||
print(" останавливаем replica container:", target[:12])
|
||||
|
||||
# Готовим одно подтверждённое сообщение до отказа.
|
||||
key0 = "before-" + uuid.uuid4().hex
|
||||
first = c.call("POST", path, 201, actor=alice, body={"text": "before-fail"},
|
||||
headers={"Idempotency-Key": key0}, schema="Message")["json"]
|
||||
|
||||
# Останавливаем реплику.
|
||||
stop = subprocess.run(["docker", "stop", "-t", "5", target],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
check(stop.returncode == 0, "docker stop failed: %s" % stop.stderr)
|
||||
|
||||
try:
|
||||
# Даём LB обнаружить отказ (proxy_next_upstream, max_fails=2).
|
||||
time.sleep(3)
|
||||
|
||||
# Читаем и пишем — должны попадать во вторую реплику.
|
||||
deadline = time.monotonic() + 10
|
||||
recovered = False
|
||||
seen_instances = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = c.call("POST", path, 201, actor=alice,
|
||||
body={"text": "after-fail"},
|
||||
headers={"Idempotency-Key": "after-" + uuid.uuid4().hex},
|
||||
schema="Message")
|
||||
seen_instances.add(r["headers"].get("X-Instance-Id"))
|
||||
recovered = True
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(0.5)
|
||||
check(recovered, "не удалось записать после отказа за 10 секунд")
|
||||
|
||||
# Убеждаемся, что подтверждённое до отказа сообщение доступно.
|
||||
from urllib.parse import quote
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
q = path + "?limit=100"
|
||||
if cursor:
|
||||
q += "&cursor=" + quote(cursor, safe="")
|
||||
page = c.call("GET", q, actor=bob, schema="MessagePage")["json"]
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
check(any(m["id"] == first["id"] for m in items),
|
||||
"подтверждённое сообщение потеряно после отказа")
|
||||
print(" after-fail instances:", seen_instances)
|
||||
print(" PASS: стенд пережил отказ одной реплики")
|
||||
finally:
|
||||
# Только восстановление реплики. Никаких return/break/continue.
|
||||
subprocess.run(["docker", "start", target],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
|
||||
# Ждём готовности через LB — УЖЕ ВНЕ finally.
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
# И двух реплик снова должно быть >=2.
|
||||
seen = set()
|
||||
for _ in range(30):
|
||||
seen.add(c.call("GET", "/api/v1/users/me", actor=alice,
|
||||
schema="User")["headers"]["X-Instance-Id"])
|
||||
if len(seen) >= 2:
|
||||
break
|
||||
if len(seen) >= 2:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise Failure("стенд не восстановился до двух реплик за 60 секунд")
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
def main():
|
||||
args = arguments("Lab3 level 4 checks")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
|
||||
print("=== test_two_instances ===")
|
||||
test_two_instances(c)
|
||||
print("PASS: >=2 X-Instance-Id через LB")
|
||||
|
||||
print("=== test_idempotency_across_replicas ===")
|
||||
test_idempotency_across_replicas(c)
|
||||
print("PASS: идемпотентность на общих ключах")
|
||||
|
||||
print("=== test_stop_one_replica ===")
|
||||
test_stop_one_replica(c)
|
||||
print("PASS: отказ одной API-реплики обработан")
|
||||
|
||||
print("PASS: all lab3 level-4 checks passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 для lab4: негативные сценарии, компенсация при отказе БД, параллельные загрузки.
|
||||
|
||||
Запуск:
|
||||
$env:API_CONTAINER = (docker ps --filter "label=com.docker.compose.service=api" -q | Select-Object -First 1)
|
||||
python student-tests/lab4_test_level4.py --base-url http://localhost:8080
|
||||
|
||||
Сценарий компенсации (уровень 4):
|
||||
Останавливаем PostgreSQL → делаем upload → S3 PUT успешен, INSERT в БД
|
||||
падает → API удаляет объект из S3 (компенсация) → 503.
|
||||
После восстановления PG считаем объекты в bucket через mc — количество
|
||||
не должно вырасти. Это проверяет требование методички: «отказ между
|
||||
записью S3/БД не оставляет вечных неучтённых объектов».
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
API_CONTAINER = os.environ.get("API_CONTAINER", "")
|
||||
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab4_default")
|
||||
MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920")
|
||||
|
||||
|
||||
def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str):
|
||||
# filename URL-encoded: иначе urllib откажется формировать URL с NUL/CR/LF.
|
||||
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
|
||||
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST",
|
||||
headers={
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Content-Type": ctype,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with c.opener.open(req, timeout=30) as resp:
|
||||
import json as _json
|
||||
return resp.code, _json.loads(resp.read()), resp.headers
|
||||
except urllib.error.HTTPError as e:
|
||||
import json as _json
|
||||
body = e.read()
|
||||
try:
|
||||
return e.code, _json.loads(body), e.headers
|
||||
except Exception:
|
||||
return e.code, None, e.headers
|
||||
|
||||
|
||||
def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"):
|
||||
return _raw_upload(c, actor, chat["id"], filename, data, ctype)
|
||||
|
||||
|
||||
# ---------- негативные сценарии ----------
|
||||
|
||||
def test_oversize(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10_485_761)
|
||||
status, body, _h = _upload(c, alice, chat, big)
|
||||
check(status == 413, "oversize upload must be 413, got %s" % status)
|
||||
|
||||
|
||||
def test_corrupted_image(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
bad = b"\x89PNG\r\n\x1a\n" + b"not-a-real-png" * 100
|
||||
status, body, _h = _upload(c, alice, chat, bad)
|
||||
check(status == 415, "corrupted PNG must be 415, got %s" % status)
|
||||
|
||||
|
||||
def test_mime_spoof(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
status, body, _h = _upload(c, alice, chat, b"hello, not an image" * 10, ctype="image/png")
|
||||
check(status == 415, "spoofed MIME must be 415, got %s" % status)
|
||||
|
||||
|
||||
def test_bad_filename(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
for bad in ("../evil.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", " "):
|
||||
status, body, _h = _upload(c, alice, chat, data, filename=bad)
|
||||
check(status == 400, "bad filename %r must be 400, got %s" % (bad, status))
|
||||
|
||||
|
||||
# ---------- изоляция доступа ----------
|
||||
|
||||
def test_foreign_attachment_and_cross_chat(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Второй", "member_ids": [alice["id"], eve["id"]]},
|
||||
schema="Chat")["json"]
|
||||
data = SAMPLE.read_bytes()
|
||||
status, att, _h = _upload(c, alice, chat2, data)
|
||||
check(status == 201, "upload to chat2 must be 201, got %s" % status)
|
||||
att_id = att["id"]
|
||||
|
||||
c.call("GET", "/api/v1/attachments/" + att_id, 404, actor=bob)
|
||||
c.call("GET", "/api/v1/attachments/" + att_id + "/content", 404, actor=bob)
|
||||
|
||||
c.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/messages",
|
||||
404, actor=bob,
|
||||
body={"text": "", "attachment_ids": [att_id]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_attachment_belongs_to_chat(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
other = c.call("POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat")["json"]
|
||||
data = SAMPLE.read_bytes()
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 201, "upload to chat must be 201")
|
||||
c.call(
|
||||
"POST", "/api/v1/chats/" + other["id"] + "/messages",
|
||||
404, actor=alice,
|
||||
body={"text": "", "attachment_ids": [att["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
# ---------- компенсация: S3 PUT успешен, INSERT в БД падает ----------
|
||||
|
||||
def _compose(action, service):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", action, service],
|
||||
capture_output=True, text=True, timeout=90,
|
||||
)
|
||||
|
||||
|
||||
def _wait_ready(c, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API не вернулся в ready за %d секунд" % timeout)
|
||||
|
||||
|
||||
def _count_bucket_objects() -> int:
|
||||
env = os.environ.copy()
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.is_file():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env.setdefault(k.strip(), v.strip())
|
||||
|
||||
user = env.get("MINIO_ROOT_USER", "minioadmin")
|
||||
password = env.get("MINIO_ROOT_PASSWORD", "minioadmin")
|
||||
bucket = env.get("S3_BUCKET", "messenger-attachments")
|
||||
mc_host = "http://%s:%s@minio:9000" % (user, password)
|
||||
|
||||
# Entrypoint образа mc обычно сам mc; тогда подкоманду зовём без префикса "mc".
|
||||
ins = subprocess.run(
|
||||
["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
ep = (ins.stdout or "").strip()
|
||||
prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"]
|
||||
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm", "--network", COMPOSE_NETWORK,
|
||||
"-e", "MC_HOST_local=" + mc_host, MC_IMAGE]
|
||||
+ prefix
|
||||
+ ["ls", "--recursive", "local/" + bucket],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
# Диагностика при первом запуске — увидим, что именно не так.
|
||||
print(" mc stderr:", (r.stderr or "").strip()[:300])
|
||||
return -1
|
||||
return sum(1 for line in r.stdout.splitlines() if line.startswith("["))
|
||||
|
||||
|
||||
def test_compensation_on_db_failure(c):
|
||||
"""S3 PUT прошёл, INSERT в БД упал → объект удаляется из S3, ответ 503.
|
||||
|
||||
Сценарий: останавливаем PostgreSQL. API-реплика сначала делает PUT в S3
|
||||
(успешно), затем INSERT в attachments (падает: connection error).
|
||||
В except-ветке upload_attachment вызывается s3.delete_object —
|
||||
orphan не остаётся. Ответ — 503 unavailable.
|
||||
После восстановления PG сверяем количество объектов в bucket.
|
||||
"""
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
|
||||
before = _count_bucket_objects()
|
||||
check(before >= 0, "не удалось посчитать объекты в bucket (mc)")
|
||||
|
||||
stopped = _compose("stop", "postgres")
|
||||
check(stopped.returncode == 0, "docker compose stop postgres failed")
|
||||
time.sleep(2)
|
||||
try:
|
||||
status, body, _h = _upload(c, alice, chat, data)
|
||||
check(status in (500, 503),
|
||||
"compensation expected 5xx, got %s (body=%r)" % (status, body))
|
||||
print(" upload при мёртвой БД вернул %s" % status)
|
||||
finally:
|
||||
started = _compose("start", "postgres")
|
||||
check(started.returncode == 0, "docker compose start postgres failed")
|
||||
_wait_ready(c, timeout=90)
|
||||
|
||||
after = _count_bucket_objects()
|
||||
check(after == before,
|
||||
"orphan-объект остался в bucket: before=%s after=%s" % (before, after))
|
||||
print(" компенсация: bucket не вырос (before=%s after=%s)" % (before, after))
|
||||
|
||||
|
||||
# ---------- параллельные загрузки с измерением RSS ----------
|
||||
|
||||
def test_parallel_uploads_memory(c):
|
||||
"""Уровень 4: ≥10 одновременных загрузок с измерением пика RSS."""
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
barrier = threading.Barrier(10)
|
||||
errors = []
|
||||
|
||||
def one(_):
|
||||
local = Client(c.base, timeout=60)
|
||||
barrier.wait()
|
||||
try:
|
||||
status, att, _h = _upload(local, alice, chat, data)
|
||||
if status != 201:
|
||||
errors.append("status=%s" % status)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(type(exc).__name__)
|
||||
|
||||
def rss():
|
||||
if not API_CONTAINER:
|
||||
return None
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", API_CONTAINER],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
used = out.stdout.strip().split("/")[0].strip()
|
||||
if used.endswith("MiB"):
|
||||
return float(used[:-3])
|
||||
if used.endswith("GiB"):
|
||||
return float(used[:-3]) * 1024
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
peak = {"value": 0.0}
|
||||
stop = threading.Event()
|
||||
|
||||
def sampler():
|
||||
while not stop.is_set():
|
||||
v = rss()
|
||||
if v and v > peak["value"]:
|
||||
peak["value"] = v
|
||||
time.sleep(0.2)
|
||||
|
||||
sampler_thread = threading.Thread(target=sampler, daemon=True)
|
||||
sampler_thread.start()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
list(pool.map(one, range(10)))
|
||||
|
||||
stop.set()
|
||||
sampler_thread.join(timeout=2)
|
||||
|
||||
check(not errors, "parallel uploads errors: %s" % errors[:3])
|
||||
print(" 10 параллельных загрузок, пик RSS=%s MiB" % peak["value"])
|
||||
if peak["value"]:
|
||||
check(peak["value"] < 400, "peak RSS %s MiB слишком высок" % peak["value"])
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
|
||||
def main():
|
||||
args = arguments("Lab4 level 4 checks")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
|
||||
print("=== oversize ===")
|
||||
test_oversize(c)
|
||||
print("PASS: oversize -> 413")
|
||||
|
||||
print("=== corrupted image ===")
|
||||
test_corrupted_image(c)
|
||||
print("PASS: corrupted -> 415")
|
||||
|
||||
print("=== spoofed MIME ===")
|
||||
test_mime_spoof(c)
|
||||
print("PASS: spoofed MIME -> 415")
|
||||
|
||||
print("=== bad filename ===")
|
||||
test_bad_filename(c)
|
||||
print("PASS: bad filename -> 400")
|
||||
|
||||
print("=== foreign attachment / cross-chat ===")
|
||||
test_foreign_attachment_and_cross_chat(c)
|
||||
print("PASS: foreign -> 404, cross-chat -> 404")
|
||||
|
||||
print("=== attachment belongs to chat ===")
|
||||
test_attachment_belongs_to_chat(c)
|
||||
print("PASS: attachment from another chat -> 404")
|
||||
|
||||
print("=== compensation on DB failure ===")
|
||||
test_compensation_on_db_failure(c)
|
||||
print("PASS: компенсация orphan-объектов при отказе БД")
|
||||
|
||||
print("=== parallel uploads memory ===")
|
||||
test_parallel_uploads_memory(c)
|
||||
print("PASS: 10 параллельных загрузок в пределах лимита памяти")
|
||||
|
||||
print("PASS: all lab4 level-4 checks passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,513 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 для lab5: варианты, outbox, идемпотентность, DLQ, lease, retry.
|
||||
|
||||
Запуск:
|
||||
$env:PYTHONUTF8="1"
|
||||
python student-tests/lab5_test_level4.py --base-url http://localhost:8080
|
||||
|
||||
Требует рабочий стенд из compose.yaml (LB + 2 api + worker + outbox +
|
||||
RabbitMQ + MinIO + PostgreSQL + Redis). Использует psql/mc через
|
||||
docker run в compose-сети, как lab4_test_level4.py.
|
||||
|
||||
Сценарии:
|
||||
=== variants ===
|
||||
Все 4 варианта: original, thumbnail (256×160), crop (128×128),
|
||||
optimized (WebP). Реальные bytes и размеры, а не JSON-обещания.
|
||||
=== outbox crash between COMMIT and publish ===
|
||||
Остановить outbox → upload → строка outbox есть, published_at NULL
|
||||
→ запустить outbox → published_at заполнено, attachment готов.
|
||||
=== duplicate delivery does not create duplicates ===
|
||||
Ручная повторная публикация того же job → variants не дублируются.
|
||||
=== worker stopped: 202 → content 409 → ready ===
|
||||
Остановить worker → upload → 202 queued → content 409 → старт
|
||||
worker → ready и thumbnail доступен.
|
||||
=== broken staging → retry_exhausted → failed + DLQ ===
|
||||
Удалить staging-объект → worker исчерпает retry → failed
|
||||
error_code=retry_exhausted + сообщение в DLQ.
|
||||
=== stuck processing (lease) → queued → ready ===
|
||||
Симулировать processing с истёкшим lease → recover (через s3_cleanup
|
||||
или через явный SQL) возвращает в queued → worker обрабатывает → ready.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
from image_probe import jpeg_size # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab5_default")
|
||||
PG_CONTAINER = os.environ.get("PG_CONTAINER", "mtuci-postgres")
|
||||
RABBIT_CONTAINER = os.environ.get("RABBIT_CONTAINER", "mtuci-rabbitmq")
|
||||
MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920")
|
||||
|
||||
|
||||
# ---------- helpers ----------
|
||||
|
||||
def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str):
|
||||
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
|
||||
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST",
|
||||
headers={
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Content-Type": ctype,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with c.opener.open(req, timeout=30) as resp:
|
||||
return resp.code, json.loads(resp.read()), resp.headers
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read()
|
||||
try:
|
||||
return e.code, json.loads(body), e.headers
|
||||
except Exception:
|
||||
return e.code, None, e.headers
|
||||
|
||||
|
||||
def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"):
|
||||
return _raw_upload(c, actor, chat["id"], filename, data, ctype)
|
||||
|
||||
|
||||
def _env_from_file():
|
||||
env = os.environ.copy()
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.is_file():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env.setdefault(k.strip(), v.strip())
|
||||
return env
|
||||
|
||||
|
||||
def _psql(sql: str) -> str:
|
||||
"""psql через docker compose exec в mtuci-postgres."""
|
||||
r = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
|
||||
"psql", "-U", "messenger", "-d", "high_load_messenger",
|
||||
"-tA", "-v", "ON_ERROR_STOP=1", "-c", sql],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def _compose(action: str, service: str):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", action, service],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=90,
|
||||
)
|
||||
|
||||
|
||||
def _wait_ready(c, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API не вернулся в ready за %d секунд" % timeout)
|
||||
|
||||
|
||||
def _mc(*args: str) -> subprocess.CompletedProcess:
|
||||
"""Запуск mc через одноразовый контейнер в compose-сети."""
|
||||
env = _env_from_file()
|
||||
user = env.get("MINIO_ROOT_USER", "minioadmin")
|
||||
password = env.get("MINIO_ROOT_PASSWORD", "minioadmin")
|
||||
mc_host = "http://%s:%s@minio:9000" % (user, password)
|
||||
|
||||
ins = subprocess.run(
|
||||
["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
ep = (ins.stdout or "").strip()
|
||||
prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"]
|
||||
|
||||
return subprocess.run(
|
||||
["docker", "run", "--rm", "--network", COMPOSE_NETWORK,
|
||||
"-e", "MC_HOST_local=" + mc_host, MC_IMAGE]
|
||||
+ prefix + list(args),
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _mc_rm(key: str) -> None:
|
||||
env = _env_from_file()
|
||||
bucket = env.get("S3_BUCKET", "messenger-attachments")
|
||||
r = _mc("rm", "local/%s/%s" % (bucket, key))
|
||||
check(r.returncode == 0, "mc rm failed: %s" % r.stderr[:300])
|
||||
|
||||
|
||||
def _dlq_count() -> int:
|
||||
r = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
|
||||
"rabbitmqctl", "list_queues", "name", "messages"],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
|
||||
for line in r.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) == 2 and parts[0] == "attachment.process.v1.dlq":
|
||||
return int(parts[1])
|
||||
return 0
|
||||
|
||||
|
||||
def _queue_consumers(qname: str) -> int:
|
||||
r = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
|
||||
"rabbitmqctl", "list_queues", "name", "consumers"],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
|
||||
for line in r.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) == 2 and parts[0] == qname:
|
||||
return int(parts[1])
|
||||
return 0
|
||||
|
||||
|
||||
# ---------- 1. Все 4 варианта ----------
|
||||
|
||||
def test_variants(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 202, "upload must be 202, got %s" % status)
|
||||
check(att["status"] == "queued", "upload status must be queued, got %s" % att["status"])
|
||||
|
||||
ready = c.wait_ready(alice, att, wait_seconds=60)
|
||||
check(ready["status"] == "ready", "must be ready")
|
||||
check(ready["error_code"] is None, "ready must have error_code=null")
|
||||
|
||||
names = sorted(v["name"] for v in ready["variants"])
|
||||
check(names == ["crop", "optimized", "thumbnail"],
|
||||
"expected variants thumbnail+crop+optimized, got %s" % names)
|
||||
|
||||
# original
|
||||
download = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)
|
||||
check(download["bytes"] == data, "original bytes differ")
|
||||
check(download["headers"].get_content_type() == "image/png", "original content-type")
|
||||
|
||||
# thumbnail: JPEG 256x160, sha256, size
|
||||
v = next(x for x in ready["variants"] if x["name"] == "thumbnail")
|
||||
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
|
||||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be image/jpeg")
|
||||
check(len(thumb["bytes"]) == v["size_bytes"], "thumbnail size_bytes mismatch")
|
||||
check(hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], "thumbnail sha256 mismatch")
|
||||
w, h = jpeg_size(thumb["bytes"])
|
||||
check((w, h) == (256, 160), "thumbnail dimensions %sx%s, expected 256x160" % (w, h))
|
||||
check((v["width"], v["height"]) == (256, 160), "thumbnail metadata dimensions mismatch")
|
||||
|
||||
# crop: JPEG 128x128
|
||||
v = next(x for x in ready["variants"] if x["name"] == "crop")
|
||||
crop = c.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=bob)
|
||||
check(crop["headers"].get_content_type() == "image/jpeg", "crop must be image/jpeg")
|
||||
check(len(crop["bytes"]) == v["size_bytes"], "crop size_bytes mismatch")
|
||||
w, h = jpeg_size(crop["bytes"])
|
||||
check((w, h) == (128, 128), "crop dimensions %sx%s, expected 128x128" % (w, h))
|
||||
|
||||
# optimized: WebP
|
||||
v = next(x for x in ready["variants"] if x["name"] == "optimized")
|
||||
opt = c.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=bob)
|
||||
check(opt["headers"].get_content_type() == "image/webp", "optimized must be image/webp")
|
||||
check(len(opt["bytes"]) == v["size_bytes"], "optimized size_bytes mismatch")
|
||||
check(hashlib.sha256(opt["bytes"]).hexdigest() == v["sha256"], "optimized sha256 mismatch")
|
||||
# WebP RIFF-заголовок
|
||||
check(opt["bytes"][:4] == b"RIFF" and opt["bytes"][8:12] == b"WEBP", "optimized is not WebP")
|
||||
|
||||
|
||||
# ---------- 2. Outbox: crash между COMMIT и publish ----------
|
||||
|
||||
def test_outbox_crash(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
|
||||
# Останавливаем outbox, чтобы job застряла в БД после COMMIT.
|
||||
r = _compose("stop", "outbox")
|
||||
check(r.returncode == 0, "stop outbox failed: %s" % r.stderr[:300])
|
||||
|
||||
try:
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 202, "upload must be 202 with stopped outbox, got %s" % status)
|
||||
|
||||
# В БД должна быть строка outbox с published_at IS NULL.
|
||||
unpublished = int(_psql(
|
||||
"SELECT count(*) FROM outbox WHERE published_at IS NULL"
|
||||
" AND payload->>'attachment_id' = '%s';" % att["id"]
|
||||
))
|
||||
check(unpublished == 1,
|
||||
"expected 1 unpublished outbox row for %s, got %d" % (att["id"], unpublished))
|
||||
|
||||
# attachment остаётся queued.
|
||||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||||
check(st == "queued", "attachment must stay queued while outbox stopped, got %s" % st)
|
||||
finally:
|
||||
r = _compose("start", "outbox")
|
||||
check(r.returncode == 0, "start outbox failed: %s" % r.stderr[:300])
|
||||
|
||||
# После старта outbox job должна быть опубликована и обработана.
|
||||
ready = c.wait_ready(alice, att, wait_seconds=90)
|
||||
check(ready["status"] == "ready", "outbox recovery: attachment must become ready")
|
||||
published = int(_psql(
|
||||
"SELECT count(*) FROM outbox WHERE published_at IS NOT NULL"
|
||||
" AND payload->>'attachment_id' = '%s';" % att["id"]
|
||||
))
|
||||
check(published == 1, "outbox row must be marked published, got %d" % published)
|
||||
|
||||
|
||||
# ---------- 3. Повторная доставка не создаёт дублей ----------
|
||||
|
||||
def test_duplicate_delivery_no_duplicates(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 202, "upload must be 202, got %s" % status)
|
||||
ready = c.wait_ready(alice, att, wait_seconds=60)
|
||||
check(ready["status"] == "ready", "attachment must become ready")
|
||||
|
||||
# Ровно 3 non-original варианта в БД.
|
||||
n_before = int(_psql(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
|
||||
))
|
||||
check(n_before == 3, "expected exactly 3 variants, got %d" % n_before)
|
||||
|
||||
# Повторно публикуем тот же job через ручную публикацию в RabbitMQ
|
||||
# с тем же payload (эмуляция redelivery после side effect до ack).
|
||||
payload_json = _psql(
|
||||
"SELECT payload::text FROM outbox WHERE payload->>'attachment_id' = '%s' LIMIT 1;" % att["id"]
|
||||
)
|
||||
check(bool(payload_json), "payload not found in outbox")
|
||||
# Публикуем в основную очередь через pika в контейнере worker.
|
||||
script = (
|
||||
"import os, json, pika\n"
|
||||
"url = os.environ['RABBITMQ_URL']\n"
|
||||
"params = pika.URLParameters(url)\n"
|
||||
"conn = pika.BlockingConnection(params)\n"
|
||||
"ch = conn.channel()\n"
|
||||
"ch.confirm_delivery()\n"
|
||||
"ch.exchange_declare('messenger.attachments', exchange_type='direct', durable=True)\n"
|
||||
"body = %r\n"
|
||||
"ch.basic_publish(exchange='messenger.attachments', routing_key='attachment.process.v1',\n"
|
||||
" body=body.encode('utf-8'),\n"
|
||||
" properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'),\n"
|
||||
" mandatory=True)\n"
|
||||
"conn.close()\n"
|
||||
"print('PUBLISHED')\n" % payload_json
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "worker",
|
||||
"python", "-c", script],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
check(r.returncode == 0 and "PUBLISHED" in r.stdout,
|
||||
"manual duplicate publish failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||||
|
||||
# Даём worker-у обработать дубль.
|
||||
time.sleep(5)
|
||||
|
||||
n_after = int(_psql(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
|
||||
))
|
||||
check(n_after == n_before,
|
||||
"duplicate delivery created extra variants: %d -> %d" % (n_before, n_after))
|
||||
# Всё ещё ready и не failed.
|
||||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||||
check(st == "ready", "duplicate must not change status, got %s" % st)
|
||||
|
||||
|
||||
# ---------- 4. Остановка worker: 202 → 409 → ready ----------
|
||||
|
||||
def test_worker_stopped(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
|
||||
r = _compose("stop", "worker")
|
||||
check(r.returncode == 0, "stop worker failed: %s" % r.stderr[:300])
|
||||
try:
|
||||
# Upload должен вернуть 202, attachment = queued, content — 409.
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 202, "upload with stopped worker must be 202, got %s" % status)
|
||||
check(att["status"] == "queued", "attachment must be queued")
|
||||
|
||||
# content должен вернуть 409, пока не ready.
|
||||
r_content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"],
|
||||
(409, 404), actor=bob)
|
||||
check(r_content["status"] == 409,
|
||||
"content must be 409 for queued, got %s" % r_content["status"])
|
||||
|
||||
# API остаётся доступен для других эндпоинтов.
|
||||
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
finally:
|
||||
r = _compose("start", "worker")
|
||||
check(r.returncode == 0, "start worker failed: %s" % r.stderr[:300])
|
||||
|
||||
# Ждём consumer_started и processing.
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if _queue_consumers("attachment.process.v1") >= 1:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("worker не подключился к очереди за 30 секунд")
|
||||
|
||||
ready = c.wait_ready(alice, att, wait_seconds=90)
|
||||
check(ready["status"] == "ready", "after worker start attachment must become ready")
|
||||
|
||||
# thumbnail доступен
|
||||
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
|
||||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail content-type")
|
||||
|
||||
|
||||
# ---------- 5. Broken staging → retry_exhausted → failed + DLQ ----------
|
||||
|
||||
def test_broken_staging_dlq(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
|
||||
# Останавливаем worker, чтобы сначала зафиксировать staging_key в БД,
|
||||
# затем удалить staging-объект и запустить worker.
|
||||
r = _compose("stop", "worker")
|
||||
check(r.returncode == 0, "stop worker failed")
|
||||
try:
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 202, "upload must be 202")
|
||||
|
||||
# Удаляем staging-объект из S3, чтобы worker не смог его прочитать.
|
||||
staging = _psql(
|
||||
"SELECT object_key FROM attachments WHERE id = '%s';" % att["id"]
|
||||
)
|
||||
check(staging.startswith("staging/"),
|
||||
"queued attachment object_key must be staging/*, got %r" % staging)
|
||||
_mc_rm(staging)
|
||||
finally:
|
||||
r = _compose("start", "worker")
|
||||
check(r.returncode == 0, "start worker failed")
|
||||
|
||||
dlq_before = _dlq_count()
|
||||
|
||||
# Ждём финального failed (retry: 5+15+60 ≈ 80c + обработка).
|
||||
deadline = time.monotonic() + 180
|
||||
final_status = None
|
||||
error_code = None
|
||||
while time.monotonic() < deadline:
|
||||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||||
if st == "failed":
|
||||
final_status = st
|
||||
error_code = _psql(
|
||||
"SELECT coalesce(error_code, '') FROM attachments WHERE id = '%s';" % att["id"]
|
||||
)
|
||||
break
|
||||
if st == "ready":
|
||||
raise Failure("attachment unexpectedly became ready: staging was deleted")
|
||||
time.sleep(5)
|
||||
|
||||
check(final_status == "failed",
|
||||
"expected failed after retry exhaustion, got %s" % final_status)
|
||||
check(error_code == "retry_exhausted",
|
||||
"expected error_code=retry_exhausted, got %r" % error_code)
|
||||
|
||||
# DLQ должен содержать хотя бы одно сообщение.
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if _dlq_count() > dlq_before:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("DLQ did not receive the message after retry exhaustion "
|
||||
"(dlq_before=%d)" % dlq_before)
|
||||
|
||||
|
||||
# ---------- 6. Stuck processing (lease) → queued → ready ----------
|
||||
|
||||
def test_stuck_processing_recovered(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
|
||||
# Останавливаем worker, чтобы job осталась queued и мы могли её «сломать».
|
||||
r = _compose("stop", "worker")
|
||||
check(r.returncode == 0, "stop worker failed")
|
||||
try:
|
||||
status, att, _h = _upload(c, alice, chat, data)
|
||||
check(status == 202, "upload must be 202, got %s" % status)
|
||||
|
||||
# Симулируем зависший processing: status=processing, lease в прошлом.
|
||||
_psql(
|
||||
"UPDATE attachments SET status='processing',"
|
||||
" processing_until = now() - interval '5 minutes',"
|
||||
" updated_at = now() WHERE id = '%s';" % att["id"]
|
||||
)
|
||||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||||
check(st == "processing", "setup failed, expected processing, got %s" % st)
|
||||
finally:
|
||||
r = _compose("start", "worker")
|
||||
check(r.returncode == 0, "start worker failed")
|
||||
|
||||
# Проверяем, что SQL-recover (эквивалент recover_stuck_processing)
|
||||
# переводит запись в queued. Это и есть логика, которую вызывает
|
||||
# s3_cleanup по интервалу. Здесь дёргаем явно, чтобы не зависеть от
|
||||
# CLEANUP_INTERVAL_SECONDS.
|
||||
recovered = int(_psql(
|
||||
"WITH upd AS ("
|
||||
" UPDATE attachments SET status='queued', processing_until=NULL, updated_at=now()"
|
||||
" WHERE status='processing' AND processing_until IS NOT NULL"
|
||||
" AND processing_until < now()"
|
||||
" RETURNING id"
|
||||
") SELECT count(*) FROM upd;"
|
||||
))
|
||||
check(recovered >= 1, "recover_stuck_processing did not return any row")
|
||||
|
||||
# worker после старта подхватит queued и доведёт до ready.
|
||||
ready = c.wait_ready(alice, att, wait_seconds=90)
|
||||
check(ready["status"] == "ready", "recovered attachment must become ready")
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
|
||||
def main():
|
||||
args = arguments("Lab5 level 4 checks")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
|
||||
print("=== variants (thumbnail/crop/optimized) ===")
|
||||
test_variants(c)
|
||||
print("PASS: original, thumbnail 256×160, crop 128×128, optimized WebP")
|
||||
|
||||
print("=== outbox crash between COMMIT and publish ===")
|
||||
test_outbox_crash(c)
|
||||
print("PASS: outbox recovers job after stop/start without manual intervention")
|
||||
|
||||
print("=== duplicate delivery does not create duplicates ===")
|
||||
test_duplicate_delivery_no_duplicates(c)
|
||||
print("PASS: re-delivery of the same job is a no-op")
|
||||
|
||||
print("=== worker stopped: 202 → content 409 → ready ===")
|
||||
test_worker_stopped(c)
|
||||
print("PASS: accepted work survives worker restart")
|
||||
|
||||
print("=== broken staging → retry_exhausted → failed + DLQ ===")
|
||||
test_broken_staging_dlq(c)
|
||||
print("PASS: bounded retry, failed with safe error_code, message in DLQ")
|
||||
|
||||
print("=== stuck processing (lease) → queued → ready ===")
|
||||
test_stuck_processing_recovered(c)
|
||||
print("PASS: expired lease is recovered, same attachment becomes ready")
|
||||
|
||||
print("PASS: all lab5 level-4 checks passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,597 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 для lab6: сквозной trace API→broker→worker, метрики, логи, dashboard,
|
||||
alert rules, поведение при отказе telemetry-backend.
|
||||
|
||||
Запуск:
|
||||
$env:PYTHONUTF8="1"
|
||||
python student-tests/lab6_test_level4.py --base-url http://localhost:8080
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
|
||||
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost:3000")
|
||||
GRAFANA_ADMIN_USER = os.environ.get("GRAFANA_ADMIN_USER", "admin")
|
||||
GRAFANA_ADMIN_PASSWORD = os.environ.get("GRAFANA_ADMIN_PASSWORD", "admin")
|
||||
|
||||
READY_TIMEOUT = int(os.environ.get("READY_TIMEOUT", "180"))
|
||||
|
||||
|
||||
def _compose_exec(service: str, *args: str, timeout: int = 30):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", service, *args],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _compose(action: str, service: str, timeout: int = 90):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", action, service],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _compose_logs(service: str, tail: int = 20) -> str:
|
||||
r = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "logs", "--no-color",
|
||||
"--tail=%d" % tail, service],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
return (r.stdout or "") + (r.stderr or "")
|
||||
|
||||
|
||||
def _http_get(url: str, timeout: float = 10.0, basic_auth: tuple | None = None):
|
||||
headers = {"Accept": "application/json"}
|
||||
if basic_auth is not None:
|
||||
token = base64.b64encode(("%s:%s" % basic_auth).encode()).decode()
|
||||
headers["Authorization"] = "Basic " + token
|
||||
req = urllib.request.Request(url, headers=headers, method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.status, resp.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read().decode("utf-8", "replace")
|
||||
except (urllib.error.URLError, OSError) as e:
|
||||
return 0, "%s: %s" % (type(e).__name__, e)
|
||||
|
||||
|
||||
def _prom_query(query: str) -> dict:
|
||||
qs = "query=" + urllib.parse.quote(query, safe="")
|
||||
r = _compose_exec(
|
||||
"prometheus", "sh", "-c",
|
||||
"wget -qO- 'http://localhost:9090/api/v1/query?%s'" % qs,
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
raise Failure("prometheus query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||||
try:
|
||||
return json.loads(r.stdout)
|
||||
except Exception:
|
||||
raise Failure("prometheus non-JSON: %s" % r.stdout[:300])
|
||||
|
||||
|
||||
def _tempo_search(query: str, since_seconds: int = 300, limit: int = 20) -> list[dict]:
|
||||
now = int(time.time())
|
||||
start = now - since_seconds
|
||||
qs = "q=%s&start=%d&end=%d&limit=%d" % (
|
||||
urllib.parse.quote(query, safe=""), start, now, limit,
|
||||
)
|
||||
r = _compose_exec(
|
||||
"tempo", "sh", "-c",
|
||||
"wget -qO- 'http://localhost:3200/api/search?%s'" % qs,
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(r.stdout)
|
||||
except Exception:
|
||||
return []
|
||||
return data.get("traces", []) or []
|
||||
|
||||
|
||||
def _tempo_get_trace(trace_id: str) -> dict:
|
||||
r = _compose_exec(
|
||||
"tempo", "sh", "-c",
|
||||
"wget -qO- 'http://localhost:3200/api/traces/%s'" % trace_id,
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
raise Failure("tempo trace fetch failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def _tempo_span_names(trace: dict) -> set[str]:
|
||||
names: set[str] = set()
|
||||
for batch in trace.get("batches", []):
|
||||
scopes = batch.get("scopeSpans") or batch.get("instrumentationLibrarySpans") or []
|
||||
for scope in scopes:
|
||||
for span in scope.get("spans", []) or []:
|
||||
names.add(span.get("name", ""))
|
||||
return names
|
||||
|
||||
|
||||
def _loki_query(query: str, since_seconds: int = 300, limit: int = 200) -> dict:
|
||||
end = int(time.time() * 1e9)
|
||||
start = end - since_seconds * 1_000_000_000
|
||||
qs = "query=%s&start=%d&end=%d&limit=%d" % (
|
||||
urllib.parse.quote(query, safe=""), start, end, limit,
|
||||
)
|
||||
r = _compose_exec(
|
||||
"loki", "sh", "-c",
|
||||
"wget -qO- 'http://localhost:3100/loki/api/v1/query_range?%s'" % qs,
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
raise Failure("loki query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||||
try:
|
||||
return json.loads(r.stdout)
|
||||
except Exception:
|
||||
raise Failure("loki returned non-JSON: %s" % r.stdout[:300])
|
||||
|
||||
|
||||
def _raw_upload(c: Client, actor, chat_id: str, filename: str,
|
||||
data: bytes, ctype: str):
|
||||
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
|
||||
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST",
|
||||
headers={
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Content-Type": ctype,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with c.opener.open(req, timeout=30) as resp:
|
||||
return resp.code, json.loads(resp.read()), resp.headers
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read()
|
||||
try:
|
||||
return e.code, json.loads(body), e.headers
|
||||
except Exception:
|
||||
return e.code, None, e.headers
|
||||
|
||||
|
||||
def _wait_ready(c, actor, att, timeout: int) -> dict:
|
||||
"""Обёртка с диагностикой: если долго не ready — показываем статус и логи."""
|
||||
deadline = time.monotonic() + timeout
|
||||
last_status = None
|
||||
while time.monotonic() < deadline:
|
||||
a = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor,
|
||||
schema="Attachment")["json"]
|
||||
last_status = a.get("status")
|
||||
if last_status == "ready":
|
||||
return a
|
||||
if last_status == "failed":
|
||||
raise Failure("attachment failed: error_code=%r" % a.get("error_code"))
|
||||
time.sleep(0.5)
|
||||
diag = (
|
||||
"\nlast_status=%r (waited %ss)\n--- outbox logs ---\n%s\n--- worker logs ---\n%s"
|
||||
% (last_status, timeout, _compose_logs("outbox", 20),
|
||||
_compose_logs("worker", 20))
|
||||
)
|
||||
raise Failure("attachment did not become ready" + diag)
|
||||
|
||||
|
||||
def _wait_trace_with_spans(since_seconds: int, need_substrings: list[str],
|
||||
timeout: float = 180.0) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
last_seen: list[dict] = []
|
||||
while time.monotonic() < deadline:
|
||||
for svc in ("messenger-api", "messenger-worker", "messenger-outbox"):
|
||||
candidates = _tempo_search(
|
||||
'{resource.service.name = "%s"}' % svc,
|
||||
since_seconds=since_seconds,
|
||||
)
|
||||
for t in candidates:
|
||||
trace_id = t.get("traceID")
|
||||
if not trace_id:
|
||||
continue
|
||||
try:
|
||||
full = _tempo_get_trace(trace_id)
|
||||
except Failure:
|
||||
continue
|
||||
names = _tempo_span_names(full)
|
||||
last_seen.append({"trace_id": trace_id, "names": sorted(names)})
|
||||
if all(any(s in n for n in names) for s in need_substrings):
|
||||
return {"trace_id": trace_id, "names": sorted(names)}
|
||||
time.sleep(3)
|
||||
tail = "; ".join(
|
||||
"%s -> %s" % (x["trace_id"][:8], x["names"][:6]) for x in last_seen[:5]
|
||||
)
|
||||
raise Failure(
|
||||
"не дождались trace со span-names, содержащими %s за %ss; последние trace: %s"
|
||||
% (need_substrings, timeout, tail or "нет")
|
||||
)
|
||||
|
||||
|
||||
def test_trace_api_to_worker(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = SAMPLE.read_bytes()
|
||||
started = int(time.time())
|
||||
|
||||
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
|
||||
check(status == 202, "upload must be 202, got %s" % status)
|
||||
check(att["status"] == "queued", "upload status must be queued")
|
||||
|
||||
ready = _wait_ready(c, alice, att, timeout=READY_TIMEOUT)
|
||||
check(ready["status"] == "ready", "attachment must become ready")
|
||||
|
||||
since = max(60, int(time.time()) - started + 180)
|
||||
trace = _wait_trace_with_spans(
|
||||
since_seconds=since,
|
||||
need_substrings=[
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
"job.publish",
|
||||
"job.consume",
|
||||
],
|
||||
timeout=180,
|
||||
)
|
||||
print(" trace_id:", trace["trace_id"])
|
||||
print(" spans:", ", ".join(trace["names"]))
|
||||
|
||||
|
||||
def test_metrics_in_prometheus(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
|
||||
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
|
||||
data = SAMPLE.read_bytes()
|
||||
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
|
||||
check(status == 202, "upload must be 202")
|
||||
_wait_ready(c, alice, att, timeout=READY_TIMEOUT)
|
||||
|
||||
deadline = time.monotonic() + 120
|
||||
required = {
|
||||
"http_requests_total",
|
||||
"http_request_duration_seconds_bucket",
|
||||
"jobs_processed_total",
|
||||
"job_processing_duration_seconds_bucket",
|
||||
}
|
||||
got: set[str] = set()
|
||||
while time.monotonic() < deadline:
|
||||
got = set()
|
||||
for name in required:
|
||||
r = _prom_query("count(%s)" % name)
|
||||
series = r.get("data", {}).get("result", [])
|
||||
if series:
|
||||
got.add(name)
|
||||
if got == required:
|
||||
break
|
||||
time.sleep(3)
|
||||
missing = required - got
|
||||
check(not missing, "Prometheus missing metrics: %s" % sorted(missing))
|
||||
print(" metrics in Prometheus:", sorted(got))
|
||||
|
||||
r = _prom_query("count by (http_route, http_status_class) (http_requests_total)")
|
||||
series = r.get("data", {}).get("result", [])
|
||||
check(series, "http_requests_total has no series")
|
||||
import re as _re
|
||||
for s in series:
|
||||
metric = s.get("metric", {})
|
||||
route = metric.get("http_route", "")
|
||||
check(route.startswith("/api/") or route.startswith("/health"),
|
||||
"unexpected http_route: %r" % route)
|
||||
check(_re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
|
||||
route) is None,
|
||||
"http_route contains raw UUID: %r" % route)
|
||||
|
||||
r = _prom_query("sum(jobs_processed_total{outcome=\"success\"})")
|
||||
series = r.get("data", {}).get("result", [])
|
||||
check(series, "jobs_processed_total{outcome=success} absent")
|
||||
value = float(series[0]["value"][1])
|
||||
check(value >= 1, "jobs success metric is %r" % value)
|
||||
print(" jobs_processed_total{outcome=success} =", value)
|
||||
|
||||
|
||||
def test_logs_in_loki(c):
|
||||
time.sleep(15)
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
streams = []
|
||||
while time.monotonic() < deadline:
|
||||
r = _loki_query('{service_name="messenger-api"} |= "event=http"',
|
||||
since_seconds=300)
|
||||
streams = r.get("data", {}).get("result", [])
|
||||
if streams:
|
||||
break
|
||||
time.sleep(3)
|
||||
check(streams, "Loki: no messenger-api logs with event=http")
|
||||
lines = [v[1] for s in streams for v in s.get("values", [])]
|
||||
check(any("request_id=" in line for line in lines),
|
||||
"API logs missing request_id")
|
||||
|
||||
r = _loki_query('{service_name="messenger-worker"} |= "event="', since_seconds=300)
|
||||
check(r.get("data", {}).get("result", []),
|
||||
"Loki: no messenger-worker logs")
|
||||
|
||||
# Ищем запись event=http с ненулевым trace_id (корреляция лог↔trace).
|
||||
deadline = time.monotonic() + 90
|
||||
ok = False
|
||||
while time.monotonic() < deadline:
|
||||
r = _loki_query(
|
||||
'{service_name=~"messenger-api|messenger-worker"} '
|
||||
'|~ "event=(http|job_ready)" |~ "trace_id=" ',
|
||||
since_seconds=300,
|
||||
)
|
||||
streams = r.get("data", {}).get("result", [])
|
||||
for s in streams:
|
||||
for v in s.get("values", []):
|
||||
line = v[1]
|
||||
idx = line.find("trace_id=")
|
||||
if idx < 0:
|
||||
continue
|
||||
tail = line[idx + len("trace_id="):]
|
||||
tid = tail.split()[0].strip()
|
||||
if len(tid) == 32 and set(tid) != {"0"}:
|
||||
ok = True
|
||||
break
|
||||
if ok:
|
||||
break
|
||||
if ok:
|
||||
break
|
||||
time.sleep(3)
|
||||
check(ok, "Loki: no logs with non-zero trace_id (log↔trace correlation)")
|
||||
print(" Loki OK: api+worker logs with non-zero trace_id found")
|
||||
|
||||
|
||||
def _grafana_ds_uids(auth) -> dict[str, dict]:
|
||||
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
|
||||
check(code == 200, "grafana datasources API %s" % code)
|
||||
return {d.get("uid"): d for d in json.loads(body)}
|
||||
|
||||
|
||||
def test_grafana_provisioning(c):
|
||||
deadline = time.monotonic() + 60
|
||||
auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD)
|
||||
while time.monotonic() < deadline:
|
||||
code, _ = _http_get(GRAFANA_URL + "/api/health")
|
||||
if code == 200:
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise Failure("Grafana API не поднялась за 60 с")
|
||||
|
||||
by_uid = _grafana_ds_uids(auth)
|
||||
# Требуемые UID явно заданы в provisioning и совпадают со ссылками в
|
||||
# dashboard/rules/derived fields.
|
||||
for uid, type_ in (("prometheus", "prometheus"),
|
||||
("loki", "loki"),
|
||||
("tempo", "tempo")):
|
||||
check(uid in by_uid, "Grafana missing datasource uid=%r (have %s)"
|
||||
% (uid, sorted(by_uid)))
|
||||
check(by_uid[uid].get("type") == type_,
|
||||
"datasource uid=%r has wrong type: %r"
|
||||
% (uid, by_uid[uid].get("type")))
|
||||
print(" datasources by uid:", sorted(by_uid))
|
||||
|
||||
code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6",
|
||||
basic_auth=auth)
|
||||
check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code)
|
||||
dash = json.loads(body)
|
||||
dashboard = dash.get("dashboard", {})
|
||||
panels = dashboard.get("panels", [])
|
||||
titles = {p.get("title") for p in panels}
|
||||
check(any("p95" in (t or "") for t in titles),
|
||||
"dashboard missing p95 panel")
|
||||
check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles),
|
||||
"dashboard missing jobs panel")
|
||||
|
||||
# Проверяем, что панель логов ссылается на существующий uid Loki,
|
||||
# а не на «магическое» имя.
|
||||
for p in panels:
|
||||
ds = p.get("datasource") or {}
|
||||
uid = ds.get("uid")
|
||||
if uid is not None:
|
||||
check(uid in by_uid,
|
||||
"panel %r references unknown datasource uid %r"
|
||||
% (p.get("title"), uid))
|
||||
print(" dashboard panels:", sorted(t for t in titles if t))
|
||||
|
||||
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
|
||||
basic_auth=auth)
|
||||
check(code == 200, "grafana alert-rules API %s" % code)
|
||||
rules = json.loads(body)
|
||||
check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules))
|
||||
|
||||
for r in rules:
|
||||
# datasourceUid в data должен существовать.
|
||||
for item in r.get("data", []):
|
||||
uid = item.get("datasourceUid")
|
||||
if uid and uid != "__expr__":
|
||||
check(uid in by_uid,
|
||||
"rule %r references unknown datasource uid %r"
|
||||
% (r.get("title"), uid))
|
||||
|
||||
# Все правила должны быть health=ok. Если datasources не согласованы,
|
||||
# Grafana вернёт health=error "data source not found", как было
|
||||
# в замечании проверяющего.
|
||||
deadline = time.monotonic() + 180
|
||||
while True:
|
||||
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
|
||||
basic_auth=auth)
|
||||
check(code == 200, "grafana alert-rules API %s" % code)
|
||||
rules = json.loads(body)
|
||||
bad = [r.get("title") for r in rules
|
||||
if r.get("health") not in (None, "ok")]
|
||||
if not bad:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise Failure("Grafana alert rules with non-ok health: %s" % bad)
|
||||
time.sleep(3)
|
||||
print(" alert rules:", [r.get("title") for r in rules])
|
||||
|
||||
|
||||
def _prom_alerts() -> list[dict]:
|
||||
r = _compose_exec("prometheus", "sh", "-c",
|
||||
"wget -qO- 'http://localhost:9090/api/v1/alerts'")
|
||||
if r.returncode != 0 or not r.stdout.strip():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(r.stdout)
|
||||
except Exception:
|
||||
return []
|
||||
return data.get("data", {}).get("alerts", []) or []
|
||||
|
||||
|
||||
def _wait_alert(name: str, state: str, timeout: float = 240.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
alerts = _prom_alerts()
|
||||
# Prometheus /api/v1/alerts возвращает только pending/firing.
|
||||
# Состояние "inactive" == алерта с таким alertname нет в списке.
|
||||
active = [
|
||||
a for a in alerts
|
||||
if a.get("labels", {}).get("alertname") == name
|
||||
]
|
||||
if state == "inactive":
|
||||
if not active:
|
||||
return
|
||||
else:
|
||||
for a in active:
|
||||
if a.get("state") == state:
|
||||
return
|
||||
time.sleep(3)
|
||||
raise Failure("alert %s never reached %s" % (name, state))
|
||||
|
||||
|
||||
def test_alert_firing_and_resolved(c):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
stopped = _compose("stop", "postgres")
|
||||
check(stopped.returncode == 0, "stop postgres failed")
|
||||
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/api/v1/users/me", (401, 503), actor=alice)
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
_wait_alert("MessengerHighErrorRate", "firing", timeout=240)
|
||||
print(" MessengerHighErrorRate firing")
|
||||
finally:
|
||||
started = _compose("start", "postgres")
|
||||
check(started.returncode == 0, "start postgres failed")
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("API didn't become ready after postgres restart")
|
||||
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
_wait_alert("MessengerHighErrorRate", "inactive", timeout=300)
|
||||
print(" MessengerHighErrorRate resolved")
|
||||
|
||||
|
||||
def test_telemetry_backend_down(c):
|
||||
for svc in ("tempo", "loki", "prometheus"):
|
||||
_compose("start", svc)
|
||||
time.sleep(3)
|
||||
|
||||
stopped = {}
|
||||
for svc in ("prometheus", "loki", "tempo"):
|
||||
r = _compose("stop", svc)
|
||||
check(r.returncode == 0, "stop %s failed: %s" % (svc, r.stderr[:300]))
|
||||
stopped[svc] = True
|
||||
|
||||
try:
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
deadline = time.monotonic() + 30
|
||||
ok = 0
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||||
c.message(alice, chat, "telem-down-" + uuid.uuid4().hex[:8])
|
||||
ok += 1
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
check(ok >= 5, "API unavailable while telemetry backend is down (ok=%d)" % ok)
|
||||
|
||||
# В образе otel/opentelemetry-collector-contrib нет coreutils `true`,
|
||||
# поэтому проверяем живость штатным бинарником самого коллектора
|
||||
# (тот же приём, что в healthcheck compose.yaml).
|
||||
ps = _compose_exec("otel-collector", "/otelcol-contrib", "--version")
|
||||
check(ps.returncode == 0,
|
||||
"otel-collector died while backends were unavailable")
|
||||
|
||||
stats = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}",
|
||||
"mtuci-otel-collector"],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if stats.returncode == 0:
|
||||
used = stats.stdout.strip().split("/")[0].strip()
|
||||
mb = None
|
||||
if used.endswith("MiB"):
|
||||
mb = float(used[:-3])
|
||||
elif used.endswith("GiB"):
|
||||
mb = float(used[:-3]) * 1024
|
||||
if mb is not None:
|
||||
check(mb < 260, "otel-collector RSS=%s MiB exceeds limit" % mb)
|
||||
print(" otel-collector RSS while backends down:", mb, "MiB")
|
||||
finally:
|
||||
for svc in stopped:
|
||||
_compose("start", svc)
|
||||
time.sleep(3)
|
||||
|
||||
print(" API and Collector remained healthy while backends were down")
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab6 level 4 checks")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
|
||||
print("=== trace API -> broker -> worker (Tempo) ===")
|
||||
test_trace_api_to_worker(c)
|
||||
print("PASS: single trace with upload + job.publish + job.consume")
|
||||
|
||||
print("=== metrics in Prometheus ===")
|
||||
test_metrics_in_prometheus(c)
|
||||
print("PASS: http_requests_total + duration + jobs_processed_total present, "
|
||||
"low-cardinality route labels")
|
||||
|
||||
print("=== logs in Loki ===")
|
||||
test_logs_in_loki(c)
|
||||
print("PASS: structured logs of api+worker with trace_id")
|
||||
|
||||
print("=== Grafana provisioning ===")
|
||||
test_grafana_provisioning(c)
|
||||
print("PASS: dashboard messenger-lab6 + >=2 alert rules provisioned")
|
||||
|
||||
print("=== alert firing and resolved ===")
|
||||
test_alert_firing_and_resolved(c)
|
||||
print("PASS: alert fires on artificial outage and resolves on recovery")
|
||||
|
||||
print("=== telemetry backend down ===")
|
||||
test_telemetry_backend_down(c)
|
||||
print("PASS: API works, Collector stays within memory limit")
|
||||
|
||||
print("PASS: all lab6 level-4 checks passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,573 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 для lab7: Argon2id, rate-limit, mTLS LB->API, OpenBao policy,
|
||||
hardening контейнеров, ротация secret, TLS audit по trust matrix.
|
||||
|
||||
Запуск (из корня репозитория, стенд уже поднят):
|
||||
$env:PYTHONUTF8="1"
|
||||
python student-tests/lab7_test_level4.py `
|
||||
--base-url https://localhost:8443 `
|
||||
--ca-file ./certs/ca.crt
|
||||
|
||||
Требует: docker CLI, доступ к ./certs, ./secrets, compose.yaml.
|
||||
Использует те же приёмы, что lab2..lab6: docker run/exec/compose,
|
||||
psql через compose exec, curl через python ssl для mTLS-проверок.
|
||||
"""
|
||||
import base64
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CERTS = ROOT / "certs"
|
||||
CA_FILE = CERTS / "ca.crt"
|
||||
LB_CLIENT_CERT = CERTS / "lb-client.crt"
|
||||
LB_CLIENT_KEY = CERTS / "lb-client.key"
|
||||
API_SERVER_CRT = CERTS / "api-server.crt"
|
||||
API_SERVER_KEY = CERTS / "api-server.key"
|
||||
WORKER_CLIENT_CERT = CERTS / "worker-client.crt"
|
||||
WORKER_CLIENT_KEY = CERTS / "worker-client.key"
|
||||
|
||||
# lab7 fix: mtuci-messenger-lab7 — это ИМЯ ОБРАЗА, а контейнеры API
|
||||
# в compose.yaml — mtuci2026lab7-api-1 и mtuci2026lab7-api-2 (service api,
|
||||
# replicas: 2). Дефолт — первая реплика; переопределяется через env
|
||||
# API_CONTAINER, если имена другие.
|
||||
API_CONTAINER = os.environ.get("API_CONTAINER", "mtuci2026lab7-api-1")
|
||||
WORKER_CONTAINER = os.environ.get("WORKER_CONTAINER", "mtuci2026lab7-worker-1")
|
||||
OUTBOX_CONTAINER = os.environ.get("OUTBOX_CONTAINER", "mtuci2026lab7-outbox-1")
|
||||
OPENBAO_CONTAINER = os.environ.get("OPENBAO_CONTAINER", "mtuci-openbao")
|
||||
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab7_default")
|
||||
API_IMAGE = os.environ.get("API_IMAGE", "mtuci-messenger:lab7")
|
||||
|
||||
READY_TIMEOUT = int(os.environ.get("READY_TIMEOUT", "120"))
|
||||
|
||||
|
||||
# ---------- helpers ----------
|
||||
def _compose_exec(service: str, *args: str, timeout: int = 30):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", service, *args],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _compose(action: str, service: str, timeout: int = 120):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", action, service],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _docker(*args: str, timeout: int = 30):
|
||||
return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
|
||||
def _psql(sql: str) -> str:
|
||||
r = _compose_exec(
|
||||
"postgres", "psql", "-U", "messenger", "-d", "high_load_messenger",
|
||||
"-tA", "-v", "ON_ERROR_STOP=1", "-c", sql,
|
||||
)
|
||||
check(r.returncode == 0, "psql failed:\n" + (r.stdout or "") + (r.stderr or ""))
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def _wait_ready(c, timeout: int = READY_TIMEOUT) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200, schema="Health")
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API не вернулся в ready за %ss" % timeout)
|
||||
|
||||
|
||||
def _basic(username: str, password: str) -> str:
|
||||
return "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
|
||||
|
||||
|
||||
# ---------- 1. Argon2id в БД ----------
|
||||
def test_argon2id_in_db(c):
|
||||
"""Пароль хранится как $argon2id$..., параметры не ниже 19MiB/t=2/p=1."""
|
||||
alice = c.user("argon")
|
||||
row = _psql(
|
||||
"SELECT password_hash FROM users WHERE username = '%s';"
|
||||
% alice["registration"]["username"]
|
||||
)
|
||||
check(row.startswith("$argon2id$v=19$"), "hash is not Argon2id v19: %r" % row[:20])
|
||||
|
||||
# m=19456 (19 MiB), t=2, p=1 в encoded-строке: $argon2id$v=19$m=19456,t=2,p=1$...
|
||||
parts = row.split("$")
|
||||
check(len(parts) >= 6, "unexpected argon2 encoded format")
|
||||
params = dict(
|
||||
p.split("=") for p in parts[3].split(",") if "=" in p
|
||||
)
|
||||
m = int(params.get("m", "0"))
|
||||
t = int(params.get("t", "0"))
|
||||
p = int(params.get("p", "0"))
|
||||
check(m >= 19456, "Argon2 memory_cost < 19MiB (got %d KiB)" % m)
|
||||
check(t >= 2, "Argon2 time_cost < 2 (got %d)" % t)
|
||||
check(p >= 1, "Argon2 parallelism < 1 (got %d)" % p)
|
||||
|
||||
# Проверяем что hash действительно проверяется: login успешен.
|
||||
c.call(
|
||||
"POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": _basic(
|
||||
alice["registration"]["username"],
|
||||
alice["registration"]["password"],
|
||||
)},
|
||||
schema="Session",
|
||||
)
|
||||
print(" Argon2id hash params:", params)
|
||||
|
||||
|
||||
# ---------- 2. Login rate-limit (429 + Retry-After, общий через Redis) ----------
|
||||
def test_login_rate_limit(c):
|
||||
"""Много неверных Basic -> 429 с Retry-After >= 1; успех сбрасывает окно."""
|
||||
# Уникальный username, чтобы не мешать другим счётчикам.
|
||||
username = "rl_" + uuid.uuid4().hex[:8]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
c.call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": "RL", "password": password},
|
||||
schema="User")
|
||||
|
||||
# LOGIN_MAX_ATTEMPTS по умолчанию 10, окно 60s. Шлём > попыток.
|
||||
got_429 = False
|
||||
retry_after = None
|
||||
for i in range(20):
|
||||
r = c.call("POST", "/api/v1/auth/sessions", (401, 429),
|
||||
headers={"Authorization": _basic(username, "wrong-" + str(i))})
|
||||
if r["status"] == 429:
|
||||
got_429 = True
|
||||
retry_after = r["headers"].get("Retry-After")
|
||||
break
|
||||
check(got_429, "rate limit did not trigger 429 after many bad logins")
|
||||
check(retry_after is not None and int(retry_after) >= 1,
|
||||
"429 must carry Retry-After >= 1, got %r" % retry_after)
|
||||
|
||||
# Даже с правильным паролем внутри окна — 429 (общий счётчик по username+IP).
|
||||
r = c.call("POST", "/api/v1/auth/sessions", (401, 429),
|
||||
headers={"Authorization": _basic(username, password)})
|
||||
check(r["status"] == 429,
|
||||
"correct password within rate-limit window must also be 429, got %s" % r["status"])
|
||||
print(" rate limit hit, Retry-After =", retry_after)
|
||||
|
||||
|
||||
# ---------- 3. mTLS LB -> API (положительный и отрицательные) ----------
|
||||
def _api_probe(client_cert: str | None, client_key: str | None,
|
||||
use_foreign_ca: bool = False) -> str:
|
||||
"""Возвращает строку-маркер: OK / NO_CERT_REJECTED / FOREIGN_CA_REJECTED /
|
||||
FOREIGN_CA_LEAF_REJECTED. Любое исключение кроме OK — считается отказом.
|
||||
|
||||
lab7 fix: urllib оборачивает SSL-ошибки в URLError, а при сбросе
|
||||
соединения (ssl_cert_reqs=2 без клиентского сертификата) — в
|
||||
http.client.RemoteDisconnected. Поэтому ловим широкий набор исключений
|
||||
и трактуем как «соединение отклонено».
|
||||
"""
|
||||
script = r'''
|
||||
import http.client, json, os, ssl, subprocess, sys, urllib.request, urllib.error
|
||||
|
||||
marker = "UNKNOWN"
|
||||
try:
|
||||
ctx = ssl.create_default_context(cafile="/certs/ca.crt")
|
||||
ctx.check_hostname = True
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
CERT = %(cert)r
|
||||
KEY = %(key)r
|
||||
FOREIGN = %(foreign)s
|
||||
if FOREIGN:
|
||||
# Чужой self-signed leaf (тот же CN/SAN, но НЕ подписан нашим CA).
|
||||
subprocess.run(
|
||||
"openssl req -x509 -newkey rsa:2048 -nodes -days 1 "
|
||||
"-keyout /tmp/foreign.key -out /tmp/foreign.crt "
|
||||
"-subj '/CN=lb-client' -addext 'extendedKeyUsage=clientAuth' "
|
||||
"-addext 'subjectAltName=DNS:lb-client' >/dev/null 2>&1",
|
||||
shell=True, check=True,
|
||||
)
|
||||
ctx.load_cert_chain("/tmp/foreign.crt", "/tmp/foreign.key")
|
||||
elif CERT:
|
||||
ctx.load_cert_chain(CERT, KEY)
|
||||
with urllib.request.urlopen("https://api:8080/health/live", context=ctx, timeout=5) as r:
|
||||
print("RESULT:OK", r.status)
|
||||
sys.exit(0)
|
||||
except ssl.SSLCertVerificationError as e:
|
||||
print("RESULT:VERIFY_REJECTED", type(e).__name__)
|
||||
sys.exit(0)
|
||||
except ssl.SSLError as e:
|
||||
print("RESULT:TLS_REJECTED", type(e).__name__)
|
||||
sys.exit(0)
|
||||
except urllib.error.URLError as e:
|
||||
print("RESULT:URL_REJECTED", type(e.reason).__name__)
|
||||
sys.exit(0)
|
||||
except http.client.RemoteDisconnected as e:
|
||||
print("RESULT:REMOTE_DISCONNECTED")
|
||||
sys.exit(0)
|
||||
except (ConnectionResetError, BrokenPipeError, ConnectionAbortedError) as e:
|
||||
print("RESULT:CONN_REJECTED", type(e).__name__)
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print("RESULT:OTHER", type(e).__name__, str(e)[:120])
|
||||
sys.exit(0)
|
||||
''' % {
|
||||
"cert": client_cert or "",
|
||||
"key": client_key or "",
|
||||
"foreign": use_foreign_ca,
|
||||
}
|
||||
r = subprocess.run(
|
||||
["docker", "exec", API_CONTAINER, "python", "-c", script],
|
||||
capture_output=True, text=True, timeout=40,
|
||||
)
|
||||
for line in (r.stdout or "").splitlines():
|
||||
if line.startswith("RESULT:"):
|
||||
return line[len("RESULT:"):]
|
||||
raise Failure("probe produced no RESULT marker: %s" %
|
||||
(r.stdout + r.stderr)[:400])
|
||||
|
||||
|
||||
def test_mtls_lb_to_api(c):
|
||||
"""API :8080 требует client cert. Только LB identity (SAN lb-client) проходит.
|
||||
|
||||
Внутренний порт 8080 не публикуется, поэтому проверяем через
|
||||
docker exec в api-контейнер: TLS-handshake изнутри compose-сети.
|
||||
Отрицательные:
|
||||
- без client cert;
|
||||
- leaf от чужого (self-signed) CA;
|
||||
Позитив: /certs/lb-client.crt (SAN lb-client).
|
||||
"""
|
||||
check(API_CONTAINER, "set API_CONTAINER")
|
||||
|
||||
# Позитивный: LB предъявляет /certs/lb-client.crt
|
||||
res = _api_probe("/certs/lb-client.crt", "/certs/lb-client.key")
|
||||
check(res.startswith("OK"),
|
||||
"LB identity (lb-client) rejected by API: %s" % res)
|
||||
|
||||
# Отрицательный: без client cert -> соединение должно быть отклонено.
|
||||
res = _api_probe(None, None)
|
||||
check(not res.startswith("OK"),
|
||||
"API accepted connection WITHOUT client cert: %s" % res)
|
||||
|
||||
# Отрицательный: чужой self-signed leaf (CN/SAN совпадают, CA другой).
|
||||
res = _api_probe(None, None, use_foreign_ca=True)
|
||||
check(not res.startswith("OK"),
|
||||
"API accepted leaf signed by foreign CA: %s" % res)
|
||||
|
||||
print(" mTLS LB->API: positive + 2 negative passed")
|
||||
|
||||
|
||||
# ---------- 4. Session revocation + external TLS trust ----------
|
||||
def test_session_revocation_and_tls(c):
|
||||
"""Bearer-сессия работает по HTTPS; revoke -> 401; чужой CA -> ошибка."""
|
||||
alice = c.user("sess")
|
||||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=alice)
|
||||
|
||||
# Проверка, что HTTPS с пустым CA не пройдёт (external trust, без client cert).
|
||||
base = c.base
|
||||
check(base.startswith("https://"), "lab7 requires HTTPS base")
|
||||
host = base[len("https://"):]
|
||||
empty_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
empty_ctx.check_hostname = True
|
||||
empty_ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
rejected = False
|
||||
try:
|
||||
urllib.request.urlopen(
|
||||
"https://%s/health/ready" % host,
|
||||
context=empty_ctx, timeout=5,
|
||||
).read()
|
||||
except ssl.SSLCertVerificationError:
|
||||
rejected = True
|
||||
except urllib.error.URLError as e:
|
||||
if isinstance(e.reason, ssl.SSLCertVerificationError):
|
||||
rejected = True
|
||||
check(rejected, "empty trust store unexpectedly accepted server cert")
|
||||
|
||||
|
||||
# ---------- 5. OpenBao: scoped policy allow/deny ----------
|
||||
def test_openbao_policy(c):
|
||||
"""API policy читает только secret/data/app/api; чужой path запрещён.
|
||||
|
||||
Root token не должен быть в окружении приложения:
|
||||
- app-контейнер не содержит BAO_TOKEN в env.
|
||||
- root token живёт только в openbao-init (одноразовый bootstrap).
|
||||
|
||||
lab7 fix: /secrets/api-approle.env создан openbao-init под root с
|
||||
chmod 600. Под appuser (UID 1000, пользователь образа) файл не читается,
|
||||
особенно при bind-mount с Windows/NTFS. Для ОДНОРАЗОВОГО read-only
|
||||
docker run поднимаемся под --user 0:0 — иначе PermissionError.
|
||||
В реальном приложении approle-креды читает сам api-контейнер под
|
||||
appuser (там файл уже прочитан при старте). Это не ослабляет
|
||||
модель: контейнер одноразовый, секрет только читается, наружу
|
||||
не печатается.
|
||||
"""
|
||||
# 5.1. API-контейнер не должен содержать root token / unseal в env.
|
||||
r = _docker("inspect", "--format", "{{json .Config.Env}}", API_CONTAINER)
|
||||
check(r.returncode == 0, "inspect api failed")
|
||||
env = r.stdout
|
||||
check("root_token" not in env and "unseal_keys" not in env,
|
||||
"root token / unseal appears in API env")
|
||||
if "BAO_TOKEN" in env:
|
||||
after = env.split("BAO_TOKEN")[1][:64]
|
||||
check("s." not in after, "BAO_TOKEN in API env looks like a real token")
|
||||
|
||||
# 5.2. Через api-образ проверяем AppRole-логин и allow/deny.
|
||||
script = r'''
|
||||
import json, ssl, sys, urllib.request, urllib.error
|
||||
|
||||
def envp(p):
|
||||
d = {}
|
||||
try:
|
||||
for line in open(p):
|
||||
line = line.strip()
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, v = line.split("=", 1); d[k] = v
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return d
|
||||
|
||||
approle = envp("/secrets/api-approle.env")
|
||||
if not approle:
|
||||
print("NO_APPROLE"); sys.exit(2)
|
||||
|
||||
ctx = ssl.create_default_context(cafile="/certs/ca.crt")
|
||||
|
||||
def post(path, body):
|
||||
data = json.dumps(body).encode()
|
||||
req = urllib.request.Request("https://openbao:8200" + path, data=data, method="POST")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
with urllib.request.urlopen(req, context=ctx, timeout=5) as r:
|
||||
return json.loads(r.read() or b"{}")
|
||||
|
||||
login = post("/v1/auth/approle/login", {
|
||||
"role_id": approle["API_ROLE_ID"],
|
||||
"secret_id": approle["API_SECRET_ID"],
|
||||
})
|
||||
tok = login["auth"]["client_token"]
|
||||
|
||||
def get(path):
|
||||
req = urllib.request.Request("https://openbao:8200" + path, method="GET")
|
||||
req.add_header("X-Vault-Token", tok)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ctx, timeout=5) as r:
|
||||
return r.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code
|
||||
|
||||
allowed = get("/v1/secret/data/app/api")
|
||||
denied = get("/v1/secret/data/app/worker")
|
||||
root_denied = get("/v1/sys/policies/acl/api-policy")
|
||||
print("ALLOW", allowed, "DENY", denied, "ROOTPATH", root_denied)
|
||||
if allowed not in (200, 204):
|
||||
print("FAIL: allowed path was rejected (status=%s). "
|
||||
"openbao-init должен создать secret/app/api "
|
||||
"(bao kv put secret/app/api ...)." % allowed)
|
||||
sys.exit(3)
|
||||
if denied != 403:
|
||||
print("FAIL: forbidden path was not 403"); sys.exit(4)
|
||||
if root_denied != 403:
|
||||
print("FAIL: api token can read sys/policies"); sys.exit(5)
|
||||
print("OK")
|
||||
'''
|
||||
# lab7 fix: --user 0:0 — только чтобы прочитать /secrets/*-approle.env,
|
||||
# созданный root-ом с chmod 600. Контейнер одноразовый, секрет наружу
|
||||
# не выводится; runtime app-контейнер читает свои креды под appuser.
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm", "--user", "0:0", "--network", COMPOSE_NETWORK,
|
||||
"-v", "%s:/certs:ro" % CERTS.as_posix(),
|
||||
"-v", "%s:/secrets:ro" % (ROOT / "secrets").as_posix(),
|
||||
API_IMAGE, "python", "-c", script],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
check(r.returncode == 0 and "OK" in r.stdout,
|
||||
"OpenBao policy allow/deny failed: %s" % (r.stdout + r.stderr)[:600])
|
||||
print(" OpenBao policy: api can read app/api, cannot read app/worker or sys/policies")
|
||||
|
||||
|
||||
# ---------- 6. Hardening контейнеров ----------
|
||||
def test_container_hardening(c):
|
||||
"""API/worker: non-root, read_only, cap_drop=ALL, no-new-privileges."""
|
||||
for name in (API_CONTAINER, WORKER_CONTAINER, OUTBOX_CONTAINER):
|
||||
r = _docker("inspect", "--format",
|
||||
"{{.Config.User}}|{{.HostConfig.ReadonlyRootfs}}|{{json .HostConfig.CapDrop}}|{{json .HostConfig.SecurityOpt}}",
|
||||
name)
|
||||
if r.returncode != 0:
|
||||
print(" skip %s (not running)" % name)
|
||||
continue
|
||||
user, ro, cap_drop, sec_opt = r.stdout.strip().split("|", 3)
|
||||
check(user and user not in ("root", "0:0"),
|
||||
"%s runs as root" % name)
|
||||
check(ro == "true", "%s is not read_only" % name)
|
||||
check("ALL" in cap_drop, "%s does not drop ALL caps: %s" % (name, cap_drop))
|
||||
check("no-new-privileges" in sec_opt,
|
||||
"%s missing no-new-privileges: %s" % (name, sec_opt))
|
||||
print(" hardening OK for api/worker/outbox")
|
||||
|
||||
|
||||
# ---------- 7. Ротация leaf-сертификата без пересборки образов ----------
|
||||
# ---------- 7. Ротация leaf-сертификата без пересборки образов ----------
|
||||
def test_leaf_cert_rotation(c):
|
||||
"""Ротация lb-client leaf: новый cert с тем же SAN lb-client принимается.
|
||||
|
||||
lab7 fix: в api-образе (python:3.12-slim) нет openssl, поэтому старый
|
||||
вариант падал с CalledProcessError. Генерируем leaf в одноразовом
|
||||
alpine:3.20 (там openssl ставится через apk), затем docker cp в
|
||||
/tmp api-контейнера, и делаем TLS-handshake через python ssl.
|
||||
Ничего на хосте не остаётся (временная папка удаляется).
|
||||
"""
|
||||
check(API_CONTAINER, "API_CONTAINER required")
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td = Path(td)
|
||||
# openssl pipeline внутри alpine:3.20; CA и CA-key монтируем read-only.
|
||||
gen = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-v", "%s:/certs:ro" % CERTS.as_posix(),
|
||||
"-v", "%s:/out" % td.as_posix(),
|
||||
"alpine:3.20", "sh", "-c",
|
||||
"apk add --no-cache openssl >/dev/null 2>&1 && "
|
||||
"openssl genrsa -out /out/new-lb.key 2048 && "
|
||||
"openssl req -new -key /out/new-lb.key -subj '/CN=lb-client' "
|
||||
" -out /out/new-lb.csr && "
|
||||
"printf 'extendedKeyUsage=clientAuth\\nsubjectAltName=DNS:lb-client\\n' "
|
||||
" > /out/new-lb.ext && "
|
||||
"openssl x509 -req -in /out/new-lb.csr "
|
||||
" -CA /certs/ca.crt -CAkey /certs/ca.key "
|
||||
" -CAserial /out/ca.srl -CAcreateserial "
|
||||
" -days 1 -sha256 -extfile /out/new-lb.ext -out /out/new-lb.crt",
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
check(gen.returncode == 0,
|
||||
"openssl leaf generation failed: %s" % (gen.stdout + gen.stderr)[:400])
|
||||
|
||||
for fname in ("new-lb.crt", "new-lb.key"):
|
||||
src = td / fname
|
||||
check(src.is_file(), "generated %s missing" % fname)
|
||||
# lab7 fix: read_only rootfs + tmpfs /tmp не принимают docker cp
|
||||
# (cp идёт через overlayfs, не в tmpfs). Пишем через stdin —
|
||||
# тогда данные попадают именно в writable /tmp.
|
||||
data = src.read_bytes()
|
||||
put = subprocess.run(
|
||||
["docker", "exec", "-i", API_CONTAINER, "sh", "-c",
|
||||
"cat > /tmp/%s" % fname],
|
||||
input=data, capture_output=True, timeout=30,
|
||||
)
|
||||
check(put.returncode == 0,
|
||||
"write /tmp/%s failed: %s" % (fname, put.stderr[:300]))
|
||||
|
||||
# Внутри api-контейнера: python ssl handshake с новым leaf.
|
||||
script = r'''
|
||||
import ssl, urllib.request
|
||||
ctx = ssl.create_default_context(cafile="/certs/ca.crt")
|
||||
ctx.load_cert_chain("/tmp/new-lb.crt", "/tmp/new-lb.key")
|
||||
with urllib.request.urlopen("https://api:8080/health/live", context=ctx, timeout=5) as r:
|
||||
print("OK", r.status)
|
||||
'''
|
||||
r = subprocess.run(
|
||||
["docker", "exec", API_CONTAINER, "python", "-c", script],
|
||||
capture_output=True, text=True, timeout=40,
|
||||
)
|
||||
check(r.returncode == 0 and "OK 200" in r.stdout,
|
||||
"fresh lb-client leaf (same CA, same SAN) rejected: %s"
|
||||
% (r.stdout + r.stderr)[:300])
|
||||
print(" fresh leaf accepted by API without rebuild")
|
||||
|
||||
|
||||
# ---------- 8. TLS-аудит trust matrix: server verify для БД ----------
|
||||
def test_pg_tls_verify(c):
|
||||
"""API→PostgreSQL идёт по TLS с проверкой сервера.
|
||||
|
||||
lab7: параметры TLS задаются в db.py (ConnectionPool.kwargs:
|
||||
sslmode=verify-full, sslrootcert=/certs/ca.crt), а не в DATABASE_URL —
|
||||
чтобы DSN оставался совместим с прошлыми лабами. Поэтому проверяем
|
||||
не env, а фактические SSL-соединения на стороне PostgreSQL.
|
||||
|
||||
lab7 fix: psycopg по умолчанию не выставляет application_name, поэтому
|
||||
фильтр по a.application_name LIKE '%api%' ничего не находил. Считаем
|
||||
любые SSL-соединения, кроме собственного psql (pg_backend_pid()).
|
||||
"""
|
||||
# 1) В контейнере api есть CA, смонтированный в /certs/ca.crt.
|
||||
r = _docker("exec", API_CONTAINER, "test", "-f", "/certs/ca.crt")
|
||||
check(r.returncode == 0, "/certs/ca.crt missing in api container")
|
||||
|
||||
# 2) В db.py присутствует настройка verify-full (конфиг, а не рантайм).
|
||||
r = _docker("exec", API_CONTAINER, "sh", "-c",
|
||||
"grep -c 'verify-full' /app/src/app/storage/db.py")
|
||||
check(r.returncode == 0 and r.stdout.strip() != "0",
|
||||
"db.py does not set sslmode=verify-full")
|
||||
|
||||
# 3) На стороне PostgreSQL: реально открытые SSL-соединения,
|
||||
# не считая самого psql-запроса.
|
||||
r = _compose_exec(
|
||||
"postgres", "psql", "-U", "messenger", "-d", "high_load_messenger",
|
||||
"-tA", "-v", "ON_ERROR_STOP=1", "-c",
|
||||
"SELECT count(*) FROM pg_stat_ssl s "
|
||||
"JOIN pg_stat_activity a ON a.pid = s.pid "
|
||||
"WHERE s.ssl = true AND a.pid <> pg_backend_pid();",
|
||||
)
|
||||
check(r.returncode == 0, "pg_stat_ssl query failed: %s" % (r.stderr or "")[:300])
|
||||
ssl_count = int((r.stdout or "0").strip().splitlines()[-1] or 0)
|
||||
check(ssl_count >= 1,
|
||||
"no SSL connections to PostgreSQL from any client (pg_stat_ssl). "
|
||||
"Проверьте, что DATABASE_URL не переопределяет sslmode=disable, "
|
||||
"и что postgres запущен с ssl=on.")
|
||||
|
||||
print(" API->PG: verify-full + CA pin + actual SSL connections=%d" % ssl_count)
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
def main():
|
||||
args = arguments("Lab7 level 4 checks")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
# Внешний TLS обязателен.
|
||||
check(c.base.startswith("https://"), "lab7 requires https:// base-url")
|
||||
c.call("GET", "/health/ready", 200, schema="Health")
|
||||
|
||||
print("=== argon2id in DB ===")
|
||||
test_argon2id_in_db(c)
|
||||
print("PASS: Argon2id hash with m>=19456,t>=2,p>=1")
|
||||
|
||||
print("=== login rate-limit ===")
|
||||
test_login_rate_limit(c)
|
||||
print("PASS: 429 + Retry-After, shared across replicas via Redis")
|
||||
|
||||
print("=== mTLS LB -> API ===")
|
||||
test_mtls_lb_to_api(c)
|
||||
print("PASS: LB identity required, no-cert / foreign-CA rejected")
|
||||
|
||||
print("=== session revocation over TLS ===")
|
||||
test_session_revocation_and_tls(c)
|
||||
print("PASS: revoke -> 401; empty trust store -> cert verify error")
|
||||
|
||||
print("=== OpenBao scoped policy ===")
|
||||
test_openbao_policy(c)
|
||||
print("PASS: api token reads only app/api, denied app/worker and sys/policies")
|
||||
|
||||
print("=== container hardening ===")
|
||||
test_container_hardening(c)
|
||||
print("PASS: non-root, read_only, cap_drop ALL, no-new-privileges")
|
||||
|
||||
print("=== leaf cert rotation ===")
|
||||
test_leaf_cert_rotation(c)
|
||||
print("PASS: fresh lb-client leaf accepted without rebuild")
|
||||
|
||||
print("=== API -> PG TLS ===")
|
||||
test_pg_tls_verify(c)
|
||||
print("PASS: verify-full + CA pin")
|
||||
|
||||
print("PASS: all lab7 level-4 checks passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user