ЛР 6 — БВТ2303 — Васюкова — на 4 #3

Closed
Vasyukova-Nat wants to merge 19 commits from Vasyukova-Nat/mtusi2026lab6:solution/BVT2303-Vasyukova into main
74 changed files with 8885 additions and 23 deletions
+30 -6
View File
@@ -5,19 +5,40 @@ APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
DATABASE_URL=postgresql://user:password@postgres:5432/high_load_messenger
POSTGRES_USER=user
POSTGRES_PASSWORD=password
POSTGRES_DB=high_load_messenger
# Redis. Пароль не задан: Redis доступен только внутри compose-сети (порт наружу не публикуется)
REDIS_URL=redis://redis:6379/0
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
# --- MinIO / S3 (lab4) ---
# Внутренний endpoint, доступен только из compose-сети.
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Администратор MinIO (используется только minio и minio-init).
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=minioadmin-secret-change-me
# Прикладные S3-credentials (ограниченные права). Используются API и cleanup.
S3_APP_KEY_ID=messenger-app
S3_APP_SECRET_KEY=messenger-app-secret-change-me
# Лимиты (контракт + lab4)
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Фоновая сверка S3↔БД (level 4)
CLEANUP_INTERVAL_SECONDS=300
# Grace period: объекты моложе N секунд не удаляются (защита in-flight uploads).
ORPHAN_GRACE_SECONDS=3600
RABBITMQ_USER=messenger
RABBITMQ_PASSWORD=messenger-rabbit-change-me
RABBITMQ_URL=amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
@@ -25,6 +46,9 @@ JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_SERVICE_NAME=messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
GRAFANA_ADMIN_PASSWORD=admin
+1
View File
@@ -10,6 +10,7 @@ certs/private/
*.jks
*.pem
*.tfstate*
*.dump
volumes/
data/
backups/
+31
View File
@@ -0,0 +1,31 @@
# --- build stage ---
FROM python:3.12-slim AS builder
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
# --- runtime stage ---
FROM python:3.12-slim
# Непривилегированный пользователь
RUN useradd --create-home --shell /bin/bash appuser
WORKDIR /app
COPY --from=builder /install /usr/local
COPY src/ ./src/
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONPATH=/app/src
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live').read()"
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--app-dir", "src"]
+5 -2
View File
@@ -4,10 +4,10 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-student load test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; test-student: own tests; load: synthetic writes"
check:
$(PYTHON) scripts/check_contract.py
@@ -18,6 +18,9 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-student:
$(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)"
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
+261 -15
View File
@@ -1,37 +1,283 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Васюкова Наталья, БВТ2303
- Целевая оценка: 4
- Commit/PR: solution/BVT2303-Vasyukova
- Использованные источники, библиотеки и помощь ИИ:
- Контракт: contracts/openapi.json, contracts/README.md, contracts/observability.md
- Инструкции: README.md, COURSE.md, CONTRIBUTING.md
- Библиотеки: fastapi==0.115.0, uvicorn[standard]==0.32.0, psycopg[binary]==3.2.3, psycopg_pool==3.2.3, redis==5.2.0, boto3==1.35.74, Pillow==11.0.0, pika==1.3.2, opentelemetry-api==1.28.2, opentelemetry-sdk==1.28.2, opentelemetry-exporter-otlp-proto-http==1.28.2, opentelemetry-instrumentation-fastapi==0.49b2, opentelemetry-instrumentation-requests==0.49b2
- ИИ: DeepSeek — помощь в структурировании кода и отчёта. Весь код проверен, отредактирован и запущен разработчиком.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Версии runtime/образов:
- Python: 3.12 (образ `python:3.12-slim`)
- FastAPI 0.115.0, uvicorn[standard] 0.32.0, psycopg[binary] 3.2.3, psycopg_pool 3.2.3, redis 5.2.0, boto3 1.35.74, Pillow 11.0.0, pika 1.3.2
- OTel SDK 1.28.2, FastAPI instrumentation 0.49b2
- PostgreSQL 18.6-alpine, Redis 7.4.1-alpine, nginx 1.27.3-alpine
- RabbitMQ 3.13.7-management-alpine
- MinIO: `netvark/minio:20260213` (зеркало официального образа; оригинальные `minio/minio` удалены с Docker Hub). CLI `mc`: `netvark/mc:20260920`.
- OTel Collector: `otel/opentelemetry-collector-contrib:0.108.0`
- Prometheus: `prom/prometheus:v2.54.1`, Loki: `grafana/loki:3.1.1`, Tempo: `grafana/tempo:2.5.0`, Grafana: `grafana/grafana:11.2.0`
- Docker Desktop (Windows)
### Запуск стека
1. Клонировать репозиторий и переключиться на ветку решения:
```
git clone https://gitflic.ru/project/vasyukova-nat/mtusi2026lab5.git
git checkout solution/BVT2303-Vasyukova
```
2. Создать `.env` из шаблона:
```
copy .env.example .env
```
3. Собрать и поднять (LB + 2 реплики API + PostgreSQL + Redis + MinIO + minio-init + rabbitmq + worker + outbox + s3_cleanup + OTel Collector + Prometheus + Loki + Tempo + Grafana):
```
docker compose --env-file .env up -d --build
```
4. Проверить готовность:
```
curl.exe -s http://localhost:8080/health/ready
```
Ожидается: `{"status":"ok"}`.
5. Grafana доступна на `http://localhost:3000` (admin/admin). Дашборд `messenger-lab6` и datasources импортируются автоматически.
Все проверки ниже выполнялись с `PYTHONUTF8=1` (Windows по умолчанию использует cp1251 и падает на чтении UTF-8 JSON):
```
$env:PYTHONUTF8="1"
```
### Проверки
`make` в среде автора недоступен (Windows без GNU Make), поэтому использованы эквивалентные прямые вызовы скриптов. Для Linux/CI рядом указаны команды `make`.
| Makefile-команда | Прямой вызов |
| --- | --- |
| `make check` | `python scripts/check_contract.py` |
| `make validate` | `python scripts/validate_openapi.py` |
| `make test` | `python tests/smoke.py --base-url http://localhost:8080` |
| `make test-persistence` | `python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8080` |
| `make test-failover` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8080` |
| `make test-student` | `python student-tests/lab6_test_level4.py --base-url http://localhost:8080` |
### Остановка и удаление учебных данных
```
docker compose stop # данные PostgreSQL, Redis, MinIO, RabbitMQ, Prometheus, Loki, Tempo, Grafana сохраняются (volumes)
docker compose down # volumes сохраняются
docker compose down -v # ТОЛЬКО если осознанно стираем данные
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Схема процессов и хранилищ:
```
┌─ api-1 (uvicorn:8080) ┐
HTTP-клиент → lb:80 ──┤ │
(nginx) └─ api-2 (uvicorn:8080) ├→ PostgreSQL 18.6 (метаданные, outbox)
├→ Redis 7.4.1 (сессии)
├→ MinIO (S3: staging + final байты)
└→ RabbitMQ (durable job queue)
outbox (publisher) → читает unpublished из PostgreSQL → публикует в RabbitMQ
worker (consumer) → забирает job из RabbitMQ → читает staging → пишет final+variants → status=ready
s3_cleanup → сверка S3↔БД, recover зависших processing
API/worker/outbox → OTLP/HTTP → otel-collector → ┬→ Tempo (traces)
├→ Loki (logs)
└→ Prometheus (metrics)
→ Grafana (дашборд, алерты)
```
Все сервисы — в одной compose-сети `mtuci2026lab6_default`. Публично опубликован только порт LB `127.0.0.1:8080:80` и Grafana `127.0.0.1:3000:3000`. Реплики `api` **не имеют `ports`** — трафик к ним идёт исключительно через LB. PostgreSQL, Redis, MinIO, RabbitMQ, OTel Collector (4317/4318/8888/8889), Prometheus (9090), Loki (3100), Tempo (3200) доступны только по внутренним именам.
Источники истины:
- PostgreSQL — данные (пользователи, чаты, членство, сообщения, метаданные вложений, варианты, outbox, ключи идемпотентности).
- Redis — серверные сессии (общие для всех реплик).
- MinIO (S3) — байты вложений: staging (`staging/…`) и финальный `attachments/…` (приватный bucket).
- RabbitMQ — durable очередь job'ов. Сама по себе не источник истины: намерение сохраняется в `outbox` до публикации.
- Tempo/Loki/Prometheus — telemetry storage. Их падение не останавливает API.
Схема БД (подробнее — `src/app/storage/db.py`, константа `SCHEMA_SQL`):
```
users(id UUID PK, username TEXT UNIQUE, display_name TEXT, password_hash TEXT, created_at TIMESTAMPTZ)
chats(id UUID PK, title TEXT, created_at TIMESTAMPTZ)
chat_members(chat_id FK→chats, user_id FK→users, PK(chat_id,user_id))
messages(id UUID PK, chat_id FK→chats, sender_id FK→users, text TEXT, created_at TIMESTAMPTZ)
+ INDEX idx_messages_chat_created(chat_id, created_at, id)
sessions(token TEXT PK, user_id FK→users, expires_at TIMESTAMPTZ) -- fallback SESSION_STORE=postgres
idempotency_keys(user_id FK→users, method TEXT, path TEXT, key TEXT,
body_hash TEXT, response_status INT, response_body JSONB, created_at TIMESTAMPTZ,
PRIMARY KEY (user_id, method, path, key))
+ INDEX idx_idempotency_created(created_at)
attachments(id UUID PK, chat_id FK→chats, owner_id FK→users,
filename TEXT, content_type TEXT, size_bytes BIGINT, sha256 TEXT,
object_key TEXT UNIQUE, status TEXT, error_code TEXT,
processing_until TIMESTAMPTZ,
created_at TIMESTAMPTZ, updated_at TIMESTAMPTZ)
+ INDEX idx_attachments_chat(chat_id)
message_attachments(message_id FK→messages, attachment_id FK→attachments, position INT,
PRIMARY KEY (message_id, attachment_id))
attachment_variants(attachment_id FK→attachments, pipeline_version INT, variant TEXT,
object_key TEXT, content_type TEXT, size_bytes BIGINT,
width INT, height INT, sha256 TEXT, created_at TIMESTAMPTZ,
PRIMARY KEY (attachment_id, pipeline_version, variant))
outbox(id BIGSERIAL PK, job_id UUID UNIQUE, payload JSONB, created_at TIMESTAMPTZ,
published_at TIMESTAMPTZ)
+ PARTIAL INDEX idx_outbox_unpublished(id) WHERE published_at IS NULL
```
Применяется идемпотентно через `init_schema()` в отдельном one-shot сервисе `migrate` (`python -m app.storage.db`). Все временные метки — `TIMESTAMPTZ`.
Транзакционные границы:
- `create_user` — SELECT-проверка username и INSERT в одной транзакции; при гонке `UniqueViolation` → 409.
- `create_chat` — INSERT `chats` + INSERT `chat_members` в одной транзакции, откат целиком.
- `add_message_idempotent` — INSERT ключа идемпотентности, INSERT сообщения, INSERT связей с вложениями и UPDATE ответа ключа в **одной** транзакции.
- `create_attachment_queued` — INSERT `attachments` (status=`queued`) + INSERT `outbox` в **одной** транзакции. Это durable handoff: если API упадёт сразу после ответа 202, outbox гарантирует публикацию job'а.
- `finalize_ready` — UPDATE `attachments` (status=`ready`, `object_key`=final) + INSERT `attachment_variants` с `ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING` в **одной** транзакции. Идемпотентно к повторной доставке.
### Дерево проекта (изменения относительно lab5 отмечены `*`)
```
mtuci2026lab6/
├── compose.yaml * # + otel-collector, prometheus, loki, tempo, grafana; tag lab6
├── Dockerfile
├── docker-entrypoint.sh # генерирует INSTANCE_ID, пишет /tmp/instance_id
├── infra/
│ ├── nginx.conf
│ ├── otel/collector.yaml * # OTLP receivers, memory_limiter, batch, exporters
│ ├── prometheus/
│ │ ├── prometheus.yaml * # scrape otel-collector:8889
│ │ └── rules/alerts.yaml *
│ ├── loki/loki.yaml *
│ ├── tempo/tempo.yaml *
│ ├── grafana/provisioning/
│ │ ├── datasources/datasources.yaml * # Prometheus/Loki/Tempo
│ │ ├── dashboards/provider.yaml *
│ │ ├── dashboards/json/messenger.json *
│ │ └── alerting/{contactpoints,policies,rules}.yaml *
│ └── runbooks/ * # high-error-rate.md, queue-lag.md, slow-upload.md
├── Makefile
├── requirements.txt * # + opentelemetry-*
├── src/app/
│ ├── main.py * # OTel init, HTTP-метрики, X-Request-Id/X-Instance-Id
│ ├── telemetry.py * # новый: единая точка init OTel + лог-формат
│ ├── outbox_worker.py * # trace context из outbox payload → job.publish span
│ ├── worker.py * # consumer span job.consume, метрики jobs_*
│ ├── api/attachments.py * # inject traceparent в outbox payload
│ ├── api/... # без изменений с lab5
│ ├── models/schemas.py
│ └── storage/ # без изменений с lab5
├── scripts/
│ ├── restart.sh / .bat
│ ├── stop-one.sh / .bat
│ └── rolling-update.sh
└── student-tests/
└── lab6_test_level4.py * # новый: traces, metrics, logs, dashboard, alerts, telemetry down
```
### Что нового в lab6 относительно lab5
- **OTel SDK** в API, worker и outbox: `init_telemetry()` настраивает TracerProvider, MeterProvider, LoggerProvider; OTLP/HTTP → `otel-collector:4318`.
- **Resource attributes**: `service.name` (messenger-api / messenger-worker / messenger-outbox), `service.version=1.6.0`, `service.instance.id`, `deployment.environment.name=lab`.
- **Логи**: единый формат с `trace_id=`/`span_id=`; `TraceContextFilter` подставляет их из активного span и в stdout, и в OTLP-экспортёр → Loki.
- **Метрики**: `http_requests_total{http.method,http.route,http.status_class}`, `http_request_duration_seconds_bucket`, `jobs_processed_total{job.type,outcome}`, `job_processing_duration_seconds_bucket`.
- **Трассы**: server span FastAPI + собственный `http.request` span в middleware; `job.publish` (PRODUCER) в outbox; `job.consume` (CONSUMER) в worker. W3C traceparent инжектится API → outbox payload → RabbitMQ headers → worker extract.
- **Collector**: `memory_limiter` (192 MiB + spike 64), `batch`, `resource` (upsert deployment.environment.name); exporters otlphttp/tempo, otlphttp/loki, prometheus:8889. Retry ограничен (`max_elapsed_time=60s`), sending_queue `queue_size=1000`.
- **Grafana provisioning**: datasources Prometheus/Loki/Tempo, dashboard `messenger-lab6`, alert rules `MessengerHighErrorRate` + `MessengerJobsBacklogGrowing` (+ `MessengerUploadLatencyHigh`).
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда | Наблюдаемый результат | Уровень |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke | `python tests/smoke.py` | PASS: lab6 public API scenario | 3 |
| Collector + хранилища 3 сигналов + Grafana в общем стенде | `docker compose ps` | otel-collector, prometheus, loki, tempo, grafana Up | 3 |
| API/worker отдают OTLP в Collector | Grafana → Explore → Prometheus/Tempo/Loki | метрики `http_*`, `jobs_*`; реальные traces; логи `messenger-api/worker` | 3 |
| Импортируемый dashboard | `test_grafana_provisioning` | `/api/dashboards/uid/messenger-lab6` → 200, панели RPS/errors/p95/jobs | 3 |
| Логи с trace_id | `lab6_test_level4.py::test_logs_in_loki` | ненулевой `trace_id=` в логах api+worker | 3/4 |
| Сохранение метаданных и сессии при рестарте | `resilience.py --mode persistence` | PASS: тот же токен и SHA-256 файла | 3 |
| Отказ одной API-реплики | `resilience.py --mode failover` | observed_instances_before=2, failed_attempts=0 | 3 |
| Сквозной trace API → broker → worker | `lab6_test_level4.py::test_trace_api_to_worker` | один trace со span'ами `POST /api/v1/chats/{chat_id}/attachments`, `job.publish`, `job.consume` | 4 |
| p50/p95/p99, error ratio, jobs, duration | `lab6_test_level4.py::test_metrics_in_prometheus` | все required метрики в Prometheus; `http_route` без UUID | 4 |
| Низкокардинальные labels | `test_metrics_in_prometheus` | `http_route` соответствует `/api/…` / `/health…`, raw UUID отсутствует | 4 |
| Alert rules минимум 2 | `test_grafana_provisioning` | `MessengerHighErrorRate`, `MessengerJobsBacklogGrowing` (+ upload latency) | 4 |
| Alert firing + resolved | `test_alert_firing_and_resolved` | stop postgres → firing, start postgres → inactive | 4 |
| Telemetry backend down | `test_telemetry_backend_down` | API пишет/читает; Collector жив, RSS < 260 MiB | 4 |
| Memory limiter / bounded retry в Collector | `infra/otel/collector.yaml` | `memory_limiter.limit_mib=192`; `retry_on_failure.max_elapsed_time=60s`; `sending_queue.queue_size=1000` | 4 |
| SLI/SLO и runbooks | `infra/prometheus/rules/alerts.yaml`, `infra/runbooks/` | error rate > 5% / 30s; retries > successes +5 / 5m; p95 upload > 2s / 3m | 4 |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Выводы тестов
```
(venv) PS ...> python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
(venv) PS ...> python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
(venv) PS ...> python tests/smoke.py --base-url http://localhost:8080
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
(venv) PS ...> python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8080
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
(venv) PS ...> python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8080
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 7, "failed_attempts": 0, "first_recovery_seconds": 4.245}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
(venv) PS ...> python student-tests/lab6_test_level4.py --base-url http://localhost:8080
=== trace API -> broker -> worker (Tempo) ===
trace_id: b5259be38ee2e0400bd833c7eb723160
spans: POST /api/v1/chats/{chat_id}/attachments, POST /api/v1/chats/{chat_id}/attachments http receive, POST /api/v1/chats/{chat_id}/attachments http send, http.request, job.consume, job.publish
PASS: single trace with upload + job.publish + job.consume
=== metrics in Prometheus ===
metrics in Prometheus: ['http_request_duration_seconds_bucket', 'http_requests_total', 'job_processing_duration_seconds_bucket', 'jobs_processed_total']
jobs_processed_total{outcome=success} = 5.0
PASS: http_requests_total + duration + jobs_processed_total present, low-cardinality route labels
=== logs in Loki ===
Loki OK: api+worker logs with non-zero trace_id found
PASS: structured logs of api+worker with trace_id
=== Grafana provisioning ===
datasources: ['loki', 'prometheus', 'tempo']
dashboard panels: ['API + worker logs', 'API RPS (total)', 'API error ratio (5xx / total)', 'HTTP latency p50/p95/p99 (s)', 'Job duration p95 (s) by outcome', 'Jobs processed by outcome']
alert rules: ['MessengerHighErrorRate', 'MessengerJobsBacklogGrowing']
PASS: dashboard messenger-lab6 + >=2 alert rules provisioned
=== alert firing and resolved ===
MessengerHighErrorRate firing
MessengerHighErrorRate resolved
PASS: alert fires on artificial outage and resolves on recovery
=== telemetry backend down ===
otel-collector RSS while backends down: 81.24 MiB
API and Collector remained healthy while backends were down
PASS: API works, Collector stays within memory limit
PASS: all lab6 level-4 checks passed
```
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Стенд: Windows 11, Docker Desktop (WSL2), 16 ГБ RAM, лимиты compose из `compose.yaml` (API 1 CPU / 512 MiB ×2; worker 1 CPU / 512 MiB; Collector 0.5 CPU / 256 MiB). Все данные синтетические; профиль — публичный `smoke.py` + `student-tests/lab6_test_level4.py`.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `docker compose stop postgres` (~30 c) | `/health/ready` → 503; `MessengerHighErrorRate` firing; после старта БД — resolved | readiness 503; alert firing; ошибки 5xx растут, видны в Prometheus и Loki с request_id | после `start postgres` readiness 200 < 30 c; alert resolved в течение окна `for: 30s` + evaluation; данные сохранены |
| `docker compose stop prometheus loki tempo` (~30 c) | API продолжает обрабатывать запросы; Collector не падает и не растёт неограниченно | API отвечает 200; `ok >= 5` запросов/сообщений за 30 c; `otel-collector --version` OK; RSS Collector < 260 MiB | после `start` backends telemetry снова появляется в Grafana; работа API и данные не пострадали |
| `docker compose stop worker` | attachment остаётся `queued`; после `start worker` → `ready` | content 409 → 202 → ready; trace `job.consume` появился после восстановления worker | ~5–10 c до `consumer_started`; ничего не потеряно (outbox + RabbitMQ) |
| `docker compose stop outbox` | upload → 202, строка outbox `published_at IS NULL`; после `start outbox` публикация и ready | unpublished=1, attachment queued; после старта ready | время до ready ≈ интервал outbox (2 c) + обработка worker |
Диагностика инцидентов:
1. **Рост 5xx при мёртвой БД.** Дашборд `Messenger` → панель «API error ratio» растёт; p95 растёт. В Loki `{service_name="messenger-api"} |= "event=db_unavailable"` — строки с `request_id`. В Tempo trace по `trace_id` показывает `http.request` → долгий child span упавший на DB. Отличие от медленного HTTP handler: при мёртвой БД ошибки идут как 503 без долгого `http.request` span; при медленном handler растёт длительность `http.request`, но `status_class` остаётся 2xx.
2. **Растущий backlog worker.** `jobs_processed_total{outcome="retry"}` опережает `success`; `MessengerJobsBacklogGrowing` firing. Проверка: `docker compose ps worker`, `rabbitmqctl list_queues name messages consumers`, Tempo → span `job.consume` со status ERROR. После восстановления worker backlog уменьшается, alert resolved.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host: HA к потере host не заявляется. 2 реплики API + LB защищают только от отказа одной реплики и от рестарта процесса.
- SLI/SLO — учебные (`5xx ratio < 5% за 1m`, `p95 upload < 2s`); не откалиброваны под production-нагрузку. Burn-rate алерты с несколькими окнами (трек 2 на 5) не реализованы — на оценку 4 достаточно пороговых правил.
- Sampling не настроен: при большом трафике объём spans/логов в Tempo/Loki будет расти; Collector `memory_limiter`/`batch` ограничивают RAM, но не объём диска. Persistence Tempo 24 ч, Loki 168 ч — учебное окно.
- Экспортёр `otlphttp` в Tempo/Loki при длительной недоступности backend может потерять часть telemetry (queue_size=1000, max_elapsed_time=60s). Потери считаются по логам Collector (level=warn) — отдельного счётчика потерь не заведено; на 4 достаточно зафиксировать ограниченное поведение.
- Абсолютные RPS/p95 в отчёте не приводятся: измерение в closed-loop на ноутбуке переносимо плохо, для 4 требуется не абсолютный throughput, а корректная связь сигналов и воспроизводимость отказов.
- Следующие шаги: sampling (parent-based / tail) и оценка потерь spans — трек 1 на 5; burn-rate multi-window alerting — трек 2 на 5.
+475
View File
@@ -0,0 +1,475 @@
x-lab-number: 6
x-required-roles:
- api
- postgres
- load_balancer
- s3
- broker
- worker
- otel_collector
- telemetry_storage
- grafana
services:
postgres:
image: postgres:18.6-alpine
container_name: mtuci-postgres
environment:
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
POSTGRES_DB: ${POSTGRES_DB:-high_load_messenger}
volumes:
- postgres-data:/var/lib/postgresql
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-high_load_messenger}"]
interval: 5s
timeout: 5s
retries: 20
start_period: 30s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
reservations:
cpus: "0.25"
memory: 256M
redis:
image: redis:7.4.1-alpine
container_name: mtuci-redis
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"]
volumes:
- redis-data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
start_period: 3s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
reservations:
cpus: "0.1"
memory: 64M
minio:
image: netvark/minio:20260213
container_name: mtuci-minio
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
MINIO_BROWSER_REDIRECT_URL: http://localhost:9001
volumes:
- minio-data:/data
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]
interval: 5s
timeout: 5s
retries: 20
start_period: 5s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
reservations:
cpus: "0.1"
memory: 128M
minio-init:
image: netvark/mc:20260920
container_name: mtuci-minio-init
depends_on:
minio:
condition: service_healthy
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}"
mc mb --ignore-existing local/"$${S3_BUCKET}"
mc anonymous set none local/"$${S3_BUCKET}" || true
if ! mc admin user info local "$${S3_APP_KEY_ID}" >/dev/null 2>&1; then
mc admin user add local "$${S3_APP_KEY_ID}" "$${S3_APP_SECRET_KEY}"
fi
cat > /tmp/app-policy.json <<'POLICY'
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::${S3_BUCKET}"]},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], "Resource": ["arn:aws:s3:::${S3_BUCKET}/*"]}
]
}
POLICY
sed -i "s/\$${S3_BUCKET}/$${S3_BUCKET}/g" /tmp/app-policy.json || true
mc admin policy create local messenger-app /tmp/app-policy.json 2>/dev/null || true
mc admin policy attach local messenger-app --user "$${S3_APP_KEY_ID}" || true
echo "minio-init done"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_APP_KEY_ID: ${S3_APP_KEY_ID:-messenger-app}
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY:-messenger-app-secret-change-me}
restart: "no"
deploy:
resources:
limits:
cpus: "0.2"
memory: 64M
migrate:
build: .
image: mtuci-messenger:lab6
command: ["python", "-m", "app.storage.db"]
environment:
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL: ${DATABASE_URL}
SESSION_STORE: ${SESSION_STORE:-redis}
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
depends_on:
postgres:
condition: service_healthy
restart: "no"
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
reservations:
cpus: "0.1"
memory: 64M
rabbitmq:
image: rabbitmq:3.13.7-management-alpine
container_name: mtuci-rabbitmq
environment:
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-messenger-rabbit-change-me}
volumes:
- rabbitmq-data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 20
start_period: 20s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
reservations:
cpus: "0.1"
memory: 128M
worker:
build: .
image: mtuci-messenger:lab6
command: ["python", "-m", "app.worker"]
environment:
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL: ${DATABASE_URL}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_APP_KEY_ID: ${S3_APP_KEY_ID}
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY}
RABBITMQ_URL: ${RABBITMQ_URL:-amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F}
OTEL_SERVICE_NAME: messenger-worker
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
depends_on:
migrate:
condition: service_completed_successfully
minio-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
otel-collector:
condition: service_started
restart: unless-stopped
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
reservations:
cpus: "0.25"
memory: 128M
outbox:
build: .
image: mtuci-messenger:lab6
command: ["python", "-m", "app.outbox_worker"]
environment:
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL: ${DATABASE_URL}
RABBITMQ_URL: ${RABBITMQ_URL:-amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F}
OTEL_SERVICE_NAME: messenger-outbox
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
otel-collector:
condition: service_started
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.3"
memory: 128M
api:
build: .
image: mtuci-messenger:lab6
environment:
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL: ${DATABASE_URL}
SESSION_STORE: ${SESSION_STORE:-redis}
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_APP_KEY_ID: ${S3_APP_KEY_ID}
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300}
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-3600}
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
depends_on:
migrate:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
minio-init:
condition: service_completed_successfully
otel-collector:
condition: service_started
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready').read()"]
interval: 5s
timeout: 3s
retries: 5
start_period: 10s
restart: unless-stopped
deploy:
replicas: 2
resources:
limits:
cpus: "1.0"
memory: 512M
reservations:
cpus: "0.25"
memory: 128M
s3_cleanup:
build: .
image: mtuci-messenger:lab6
command: ["python", "-m", "app.storage.s3_cleanup"]
environment:
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL: ${DATABASE_URL}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_APP_KEY_ID: ${S3_APP_KEY_ID}
S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY}
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300}
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-3600}
depends_on:
migrate:
condition: service_completed_successfully
minio-init:
condition: service_completed_successfully
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.3"
memory: 256M
reservations:
cpus: "0.05"
memory: 64M
lb:
image: nginx:1.27.3-alpine
container_name: mtuci-lb
ports:
- "127.0.0.1:8080:80"
volumes:
- ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
api:
condition: service_healthy
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
reservations:
cpus: "0.1"
memory: 32M
# ---------- Observability: lab6 ----------
otel-collector:
image: otel/opentelemetry-collector-contrib:0.108.0
container_name: mtuci-otel-collector
command: ["--config=/etc/otel/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro
depends_on:
- tempo
- loki
- prometheus
# Порты OTLP наружу не публикуем: API/worker/outbox в той же сети.
# health-порт тоже внутри compose-сети.
healthcheck:
test: ["CMD", "/otelcol-contrib", "--version"]
interval: 10s
timeout: 3s
retries: 5
start_period: 10s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
reservations:
cpus: "0.1"
memory: 64M
prometheus:
image: prom/prometheus:v2.54.1
container_name: mtuci-prometheus
command:
- "--config.file=/etc/prometheus/prometheus.yaml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=15d"
- "--web.listen-address=0.0.0.0:9090"
volumes:
- ./infra/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- prometheus-data:/prometheus
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
reservations:
cpus: "0.1"
memory: 128M
loki:
image: grafana/loki:3.1.1
container_name: mtuci-loki
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
reservations:
cpus: "0.1"
memory: 128M
tempo:
image: grafana/tempo:2.5.0
container_name: mtuci-tempo
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo-data:/var/tempo
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
reservations:
cpus: "0.1"
memory: 128M
grafana:
image: grafana/grafana:11.2.0
container_name: mtuci-grafana
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
GF_AUTH_DISABLE_LOGIN_FORM: "true"
GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor"
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro
- grafana-data:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
depends_on:
- prometheus
- loki
- tempo
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.3"
memory: 256M
reservations:
cpus: "0.05"
memory: 64M
volumes:
postgres-data:
name: mtuci2026lab5_postgres-data
redis-data:
name: mtuci2026lab5_redis-data
minio-data:
name: mtuci2026lab5_minio-data
rabbitmq-data:
name: mtuci2026lab5_rabbitmq-data
prometheus-data:
name: mtuci2026lab6_prometheus-data
loki-data:
name: mtuci2026lab6_loki-data
tempo-data:
name: mtuci2026lab6_tempo-data
grafana-data:
name: mtuci2026lab6_grafana-data
networks:
default:
name: mtuci2026lab6_default
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
# X-Instance-Id: если INSTANCE_ID не задан снаружи, генерируем уникальный
# для этого контейнера при старте. Кладём его в файл /tmp/instance_id,
# чтобы stop-one.bat/.sh могли сопоставить alias и контейнер.
if [ -z "${INSTANCE_ID:-}" ]; then
INSTANCE_ID=$(python -c "import uuid; print(uuid.uuid4().hex[:12])")
export INSTANCE_ID
fi
echo "event=instance_id instance_id=${INSTANCE_ID}"
echo -n "${INSTANCE_ID}" > /tmp/instance_id
exec "$@"
+58
View File
@@ -0,0 +1,58 @@
=== Лабораторная 1. Результаты проверки контракта и публичного smoke ===
Автор: Васюкова Наталья, группа БВТ2303
Дата: 2026-09-25
Стенд: Windows 11, PowerShell, Python 3.14.5 (venv), Docker Desktop
Образ: mtusi-messenger:lab1 (python:3.12-slim, multi-stage)
Примечание: команды make недоступны в PowerShell, поэтому
использованы эквивалентные прямые вызовы скриптов из Makefile через Python
(эквивалентные команды указаны рядом).
--------------------------------------------------------------------
1. make check -> python scripts/check_contract.py
--------------------------------------------------------------------
$ python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
Наблюдение: структура репозитория целостна (скелет не изменён): lab.json и версия в openapi.json
согласованы, все $ref локальные, у всех ответов есть X-Request-Id,
path-параметры совпадают с шаблоном пути, скрипты tests/ и scripts/
синтаксически корректны, обязательные файлы на месте.
--------------------------------------------------------------------
2. make validate -> python scripts/validate_openapi.py
--------------------------------------------------------------------
$ python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
Наблюдение: контракт contracts/openapi.json прошёл полную валидацию по OpenAPI 3.1,
примеры из contracts/examples.json соответствуют схемам компонентов.
--------------------------------------------------------------------
3. make test -> python tests/smoke.py --base-url http://localhost:8080
--------------------------------------------------------------------
$ python tests/smoke.py --base-url http://localhost:8080
PASS: lab1 public API scenario; infrastructure and grading evidence remain separate.
Наблюдение: сквозной публичный сценарий ЛР1 пройден. Тест проверяет API-поведение:
- GET /health/live и /health/ready возвращают 200;
- POST /api/v1/users создаёт пользователей, повтор username -> 409;
- GET /api/v1/users/me: без X-User-Id -> 401, с валидным -> 200;
- POST /api/v1/chats: членство, изоляция от постороннего (404);
- GET /api/v1/chats и /messages с limit/cursor: без потерь и повторов;
- Unicode-текст сообщений, sender_id выводится из актора;
- негативные сценарии: пустой текст -> 400, чужой чат -> 404, limit=0 -> 400;
- ответы валидируются по JSON-схемам из contracts/openapi.json.
Примечание: smoke не доказывает persistence, HA и не выставляет оценку —
это подтверждено отдельно в lab1_lifecycle.txt.
--------------------------------------------------------------------
ИТОГ
--------------------------------------------------------------------
Все три проверки пройдены.
+155
View File
@@ -0,0 +1,155 @@
=== Лабораторная 1. Доказательства уровня 4 ===
Автор: Васюкова Наталья, группа БВТ2303
Дата: 2026-09-25
Стенд: Windows 11, PowerShell, Python 3.12 (venv), Docker Desktop
Образ: mtusi-messenger:lab1 (python:3.12-slim, multi-stage)
Commit: solution/BVT2303-Vasyukova
--------------------------------------------------------------------
1. Собственные негативные тесты
--------------------------------------------------------------------
$ python student-tests/test_negative.py --base-url http://localhost:8080
PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, wrong type,
empty text, foreign chat, pagination, bad limit, unknown user)
Наблюдение: проверены сценарии — повтор username -> 409; malformed JSON -> 400;
лишние поля -> 400; неверный тип поля -> 400; пустое/пробельное сообщение -> 400;
доступ к чужому чату (GET чата, GET сообщений, POST сообщения) -> 404;
обход нескольких страниц cursor-пагинации без потерь, дублей и изменения порядка;
limit=0 и limit=101 -> 400; неизвестный UUID в X-User-Id -> 401.
Все ответы об ошибках соответствуют формату contracts/openapi.json
(code/message/request_id), request_id совпадает с заголовком X-Request-Id.
--------------------------------------------------------------------
2. Конкурентный тест: 100 сообщений от 10 клиентов
--------------------------------------------------------------------
$ python student-tests/test_concurrency.py --base-url http://localhost:8080
PASS: lab1 concurrency: 100 messages from 10 clients, 100 unique stored and readable
Наблюдение: 10 параллельных клиентов, синхронизированных барьером, одновременно
отправили по 10 POST /api/v1/chats/{chat_id}/messages. Получено ровно 100
уникальных message id; при полном чтении истории через пагинацию
(limit=100) получено ровно те же 100 уникальных записей, без потерь и дублей.
Это подтверждает атомарность операций MemoryStorage под threading.RLock.
--------------------------------------------------------------------
3. Единая команда запуска собственных тестов
--------------------------------------------------------------------
$ python student-tests/run_all.py --base-url http://localhost:8080
=== test_negative.py ===
PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, wrong type, empty text, foreign chat, pagination, bad limit, unknown user)
=== test_concurrency.py ===
PASS: lab1 concurrency: 100 messages from 10 clients, 100 unique stored and readable
PASS: all student tests (level 4) passed
(Эквивалент через Makefile: make test-student BASE_URL=http://localhost:8080)
--------------------------------------------------------------------
4. Обработка SIGTERM
--------------------------------------------------------------------
В терминале 1:
$ docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
В терминале 2:
Цикл опроса TCP-порта 8080 (TcpClient, жёсткий таймаут 500 мс):
```
while ($true) {
$t = Get-Date -Format "HH:mm:ss.fff"
$client = New-Object System.Net.Sockets.TcpClient
try {
$iar = $client.BeginConnect("127.0.0.1", 8080, $null, $null)
$ok = $iar.AsyncWaitHandle.WaitOne(500)
if ($ok -and $client.Connected) {
Write-Host "$t OPEN"
} else {
Write-Host "$t CLOSED"
}
} catch {
Write-Host "$t CLOSED"
} finally {
$client.Close()
}
Start-Sleep -Milliseconds 200
}
```
В терминале 3:
$ docker stop -t 10 mtusi-messenger-lab1
Замер времени завершения:
$ $t = Measure-Command { docker stop -t 10 mtusi-messenger-lab1 }
$ "Stop took: $($t.TotalSeconds) s"
Stop took: 0.7416883 s
Фрагмент вывода терминала 2:
02:06:06.269 OPEN
02:06:06.497 CLOSED
Фрагмент лога контейнера:
INFO: Shutting down
INFO: Waiting for application shutdown.
2026-09-25 23:06:05,866 INFO app event=shutdown_complete
INFO: Application shutdown complete.
INFO: Finished server process [1]
Наблюдение: контейнер завершился за ~0.74 с (существенно быстрее grace-period 10 с).
Порт закрылся (первый CLOSED) через ~0.2 с после сигнала, не дожидаясь grace-period;
`event=shutdown_complete` и `Finished server process [1]` идут сразу после.
Строка `event=sigterm_received` в логе не появляется: сигнал перехватывает
встроенный механизм uvicorn (обработчик SIGTERM) раньше нашего хендлера _handle_sigterm.
--------------------------------------------------------------------
5. Логи без содержимого сообщений
--------------------------------------------------------------------
Пример строк лога контейнера (meta-only):
2026-09-25 ... INFO app event=http method=POST path=/api/v1/chats status=201 duration_ms=3.12 request_id=...
2026-09-25 ... INFO app event=http method=POST path=/api/v1/chats/<uuid>/messages status=201 duration_ms=2.01 request_id=...
2026-09-25 ... INFO app event=startup app_env=lab
2026-09-25 ... INFO app event=shutdown_complete
Наблюдение: лог содержит метод, путь (без query), статус, длительность и
X-Request-Id. Текст сообщений, значения X-User-Id, пароли и токены в лог
не попадают. Ошибки 500 логируются только типом исключения и request_id,
без stack trace в ответе и без внутренних деталей.
--------------------------------------------------------------------
6. Конфигурация контейнера (уровень 4)
--------------------------------------------------------------------
- Dockerfile: базовый образ python:3.12-slim (multi-stage), зависимости
закреплены (fastapi==0.115.0, uvicorn[standard]==0.32.0).
- Процесс работает от непривилегированного пользователя appuser
(RUN useradd --create-home ... ; USER appuser).
- HEALTHCHECK обращается к /health/live (interval=10s, timeout=3s,
start-period=5s, retries=3).
- .dockerignore исключает .env, secrets, certs, volumes, backups,
*.key/*.pem/*.p12/*.pfx/*.jks, .venv, __pycache__, node_modules,
evidence, .DS_Store. В образ не копируются локальные секреты.
Проверка:
$ docker inspect --format "{{.Config.User}}" mtusi-messenger-lab1
appuser
--------------------------------------------------------------------
ИТОГ
--------------------------------------------------------------------
Результаты:
- собственные негативные тесты — PASS;
- конкурентный тест 100/10 — PASS, 100 уникальных сообщений;
- единая команда запуска test-student — PASS;
- SIGTERM: контейнер останавливается за ~0.74 с, порт закрывается через
~0.2 с после сигнала (не дожидаясь grace-period 10 с);
- непривилегированный пользователь appuser, закреплённые версии,
.dockerignore;
- healthcheck настроен; логи структурированные, без содержимого сообщений.
+58
View File
@@ -0,0 +1,58 @@
=== Лабораторная 1. Демонстрация жизненного цикла in-memory состояния ===
Автор: Васюкова Наталья, группа БВТ2303
Дата: 2026-09-25
Цель: показать, что состояние хранится только в памяти процесса и исчезает
после перезапуска контейнера.
--------------------------------------------------------------------
1. До перезапуска: пользователь существует
--------------------------------------------------------------------
Пользователь создан через Swagger. Получен его id.
Проверка:
$ curl.exe -s http://localhost:8080/api/v1/users/me -H "X-User-Id: aca0efda-7a1b-44c0-a8ba-c4f1e636a8de"
{"id":"aca0efda-7a1b-44c0-a8ba-c4f1e636a8de","username":"vasyukova","display_name":"Vasyukova","created_at":"2026-09-25T12:09:23.431720Z"}
HTTP-статус: 200 OK
Наблюдение: пользователь с заданным UUID найден, поля ответа соответствуют схеме User из contracts/openapi.json.
--------------------------------------------------------------------
2. Остановка контейнера
--------------------------------------------------------------------
В терминале с запущенным контейнером:
Ctrl+C.
Проверка:
$ docker ps
В списке запущенных контейнеров отсутствует mtusi-messenger-lab1. Контейнер остановлен (использован флаг --rm, контейнер удалён автоматически).
--------------------------------------------------------------------
3. Повторный запуск контейнера
--------------------------------------------------------------------
$ docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
Uvicorn запущен на http://0.0.0.0:8080, контейнер готов принимать запросы.
--------------------------------------------------------------------
4. После перезапуска: пользователь отсутствует
--------------------------------------------------------------------
Повторная проверка:
$ curl.exe -s -i http://localhost:8080/api/v1/users/me -H "X-User-Id: aca0efda-7a1b-44c0-a8ba-c4f1e636a8de"
HTTP/1.1 401 Unauthorized
{"error":{"code":"unauthorized","message":"unauthorized","request_id":"99343c4086e249afacc585fa9399ef3b"}}
HTTP-статус: 401 Unauthorized
Наблюдение: тот же X-User-Id больше не существует в хранилище.
Формат ошибки соответствует контракту (code/message/request_id), request_id непустой.
--------------------------------------------------------------------
ВЫВОД
--------------------------------------------------------------------
После перезапуска пользователь исчез. Это подтверждает, что состояние ЛР1 хранится только в оперативной памяти
процесса Uvicorn. Volumes, БД, файлы состояния не используются. Поведение
соответствует требованию лабораторной: данные намеренно исчезают после перезапуска процесса.
+24
View File
@@ -0,0 +1,24 @@
{
"lab": 1,
"track": 1,
"variant": "after",
"profile": {
"history_sizes": [100, 1000, 5000],
"concurrency_levels": [1, 4, 16],
"duration_seconds": 10,
"note": "closed-loop; клиент ограничивает throughput; RSS — docker stats; sort on read REMOVED"
},
"points": [
{"history_size": 100, "concurrency": 1, "duration_s": 10.357, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2064.323, "p95": 2078.936, "p99": 2078.936}, "rss_mb": 46.01},
{"history_size": 100, "concurrency": 4, "duration_s": 10.379, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.927, "latency_ms": {"p50": 2061.138, "p95": 2090.765, "p99": 2093.1}, "rss_mb": 35.47},
{"history_size": 100, "concurrency": 16, "duration_s": 10.522, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.603, "latency_ms": {"p50": 2065.596, "p95": 2084.327, "p99": 2090.339}, "rss_mb": 35.54},
{"history_size": 1000, "concurrency": 1, "duration_s": 10.341, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2070.498, "p95": 2076.896, "p99": 2076.896}, "rss_mb": 35.91},
{"history_size": 1000, "concurrency": 4, "duration_s": 10.383, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.926, "latency_ms": {"p50": 2061.585, "p95": 2085.476, "p99": 2089.531}, "rss_mb": 36.91},
{"history_size": 1000, "concurrency": 16, "duration_s": 10.571, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.568, "latency_ms": {"p50": 2075.703, "p95": 2092.981, "p99": 2102.494}, "rss_mb": 37.53},
{"history_size": 5000, "concurrency": 1, "duration_s": 10.296, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.486, "latency_ms": {"p50": 2043.537, "p95": 2086.686, "p99": 2086.686}, "rss_mb": 41.6},
{"history_size": 5000, "concurrency": 4, "duration_s": 10.324, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.937, "latency_ms": {"p50": 2057.056, "p95": 2065.708, "p99": 2068.095}, "rss_mb": 42.15},
{"history_size": 5000, "concurrency": 16, "duration_s": 10.747, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.444, "latency_ms": {"p50": 2072.015, "p95": 2116.62, "p99": 2124.503}, "rss_mb": 43.08}
]
}
@@ -0,0 +1,24 @@
{
"lab": 1,
"track": 1,
"variant": "before",
"profile": {
"history_sizes": [100, 1000, 5000],
"concurrency_levels": [1, 4, 16],
"duration_seconds": 10,
"note": "closed-loop; клиент ограничивает throughput; RSS — docker stats; sort on read ENABLED"
},
"points": [
{"history_size": 100, "concurrency": 1, "duration_s": 10.355, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2066.139, "p95": 2085.291, "p99": 2085.291}, "rss_mb": 34.35},
{"history_size": 100, "concurrency": 4, "duration_s": 10.415, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.92, "latency_ms": {"p50": 2073.966, "p95": 2088.797, "p99": 2091.95}, "rss_mb": 34.83},
{"history_size": 100, "concurrency": 16, "duration_s": 10.589, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.555, "latency_ms": {"p50": 2071.131, "p95": 2127.922, "p99": 2134.262}, "rss_mb": 35.44},
{"history_size": 1000, "concurrency": 1, "duration_s": 10.372, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.482, "latency_ms": {"p50": 2076.358, "p95": 2082.009, "p99": 2082.009}, "rss_mb": 36.84},
{"history_size": 1000, "concurrency": 4, "duration_s": 10.381, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.927, "latency_ms": {"p50": 2062.515, "p95": 2094.389, "p99": 2106.044}, "rss_mb": 36.71},
{"history_size": 1000, "concurrency": 16, "duration_s": 10.58, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.562, "latency_ms": {"p50": 2076.528, "p95": 2101.717, "p99": 2115.533}, "rss_mb": 38.19},
{"history_size": 5000, "concurrency": 1, "duration_s": 10.39, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.481, "latency_ms": {"p50": 2076.522, "p95": 2097.586, "p99": 2097.586}, "rss_mb": 53.12},
{"history_size": 5000, "concurrency": 4, "duration_s": 10.494, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.906, "latency_ms": {"p50": 2086.895, "p95": 2108.007, "p99": 2108.871}, "rss_mb": 42.75},
{"history_size": 5000, "concurrency": 16, "duration_s": 10.701, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.476, "latency_ms": {"p50": 2090.597, "p95": 2183.025, "p99": 2188.819}, "rss_mb": 44.57}
]
}
+167
View File
@@ -0,0 +1,167 @@
# Трек 1. Предел in-memory архитектуры — сводка. Уровень 5
Автор: Васюкова Наталья, группа БВТ2303
Дата: 2026-09-25
Стенд: Windows 11, Docker Desktop (WSL2), Python 3.12, образ python:3.12-slim.
Дата: 2026-09-25.
Commit: solution/BVT2303-Vasyukova.
Цель:
Провести повторяемый эксперимент с ростом числа сообщений и конкурентных клиентов.
Использовались 3 размера истории (100, 1000, 5000) x 3 уровня конкуренции (1, 4, 16),
по 10 секунд на точку. Профиль: closed-loop чтение GET /api/v1/chats/{id}/messages?limit=100.
Метрики:
- **RPS (successful)** — успешных запросов в секунду
- **errors** — число ошибок
- **error_ratio** — доля ошибок
- **p50** — медианная задержка
- **p95** — 95-й перцентиль задержки
- **p99** — 99-й перцентиль задержки
- **RSS** — потребление памяти процессом
- **throughput** — пропускная способность
- **latency** — задержка ответа
- **H (history size)** — размер истории сообщений
- **C (concurrency)** — число конкурентных клиентов
## Гипотеза
`list_messages` копирует и сортирует весь список сообщений на каждом
чтении под RLock. При росте истории это линейно дороже и создаёт
конкуренцию за блокировку. Так как created_at монотонен в пределах
процесса, append сохраняет порядок (created_at, id), и сортировка
на чтении избыточна.
Изменение: убрать sort в list_messages, хранить список как есть.
## Ход исследования
Исследование проводится в 2 шага:
1. Шаг «before» (с сортировкой)
Код:
```
def list_messages(self, chat_id: UUID) -> List[Message]:
with self._lock:
items = list(self._messages.get(chat_id, []))
items.sort(key=lambda m: (m.created_at, str(m.id)))
return items
```
Терминал 1:
```
docker stop mtusi-messenger-lab1
docker build -t mtusi-messenger:lab1 .
docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
```
Терминал 2:
```
python student-tests/test_load_track1.py --base-url http://localhost:8080
Copy-Item evidence/lab1_track1_load.json evidence/lab1_track1_load_before.json
```
2. Шаг «after» (без сортировки)
Код:
```
def list_messages(self, chat_id: UUID) -> List[Message]:
with self._lock:
# Без пересортировки: список уже в порядке (created_at, id).
return list(self._messages.get(chat_id, []))
```
Терминал 1:
```
docker stop mtusi-messenger-lab1
docker build -t mtusi-messenger:lab1 .
docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1
```
Терминал 2:
```
python student-tests/test_load_track1.py --base-url http://localhost:8080
Copy-Item evidence/lab1_track1_load.json evidence/lab1_track1_load_after.json
```
## Результаты
Во всех точках errors = 0, error_ratio = 0.0.
### p95, мс (before / after)
| H | C=1 | C=4 | C=16 |
|-------|---------------------|---------------------|---------------------|
| 100 | 2085.3 / 2078.9 | 2088.8 / 2090.8 | 2127.9 / 2084.3 |
| 1000 | 2082.0 / 2076.9 | 2094.4 / 2085.5 | 2101.7 / 2093.0 |
| 5000 | 2097.6 / 2086.7 | 2108.0 / 2065.7 | 2183.0 / 2116.6 |
### p99, мс (before / after)
| H | C=1 | C=4 | C=16 |
|-------|---------------------|---------------------|---------------------|
| 100 | 2085.3 / 2078.9 | 2092.0 / 2093.1 | 2134.3 / 2090.3 |
| 1000 | 2082.0 / 2076.9 | 2106.0 / 2089.5 | 2115.5 / 2102.5 |
| 5000 | 2097.6 / 2086.7 | 2108.9 / 2068.1 | 2188.8 / 2124.5 |
### RPS (успешные, before / after)
| H | C=1 | C=4 | C=16 |
|-------|---------------------|---------------------|---------------------|
| 100 | 0.483 / 0.483 | 1.920 / 1.927 | 7.555 / 7.603 |
| 1000 | 0.482 / 0.483 | 1.927 / 1.926 | 7.562 / 7.568 |
| 5000 | 0.481 / 0.486 | 1.906 / 1.937 | 7.476 / 7.444 |
### RSS, МБ (before / after)
| H | C=1 | C=4 | C=16 |
|-------|---------------------|---------------------|---------------------|
| 100 | 34.35 / 46.01 | 34.83 / 35.47 | 35.44 / 35.54 |
| 1000 | 36.84 / 35.91 | 36.71 / 36.91 | 38.19 / 37.53 |
| 5000 | 53.12 / 41.60 | 42.75 / 42.15 | 44.57 / 43.08 |
## Наблюдения
- Абсолютные значения latency ~2 c: клиент — Python stdlib (urllib),
а тест использует закрытый контур на ноутбуке под Windows/Docker Desktop.
Это ограничение клиента и среды, а не только сервера.
- RPS растёт с конкуренцией (≈0.48 → ≈1.9 → ≈7.5) — узкое место в
клиентском контуре, а не в API.
- RSS растёт с историей (after: 35.5 → 37.5 → 43.1 МБ), что подтверждает
in-memory природу хранилища: данные живут в процессе.
- `after` ниже по RSS в большинстве точек: H=1000 — 35.9–37.5 против
36.7–38.2; H=5000/C=4 — 42.15 против 42.75; H=5000/C=16 — 43.08
против 44.57. Единичный выброс H=100/C=1 (46.01 после против 34.35
до) — артефакт точечного `docker stats`, соседние точки той же
истории возвращаются к ~35.5 МБ.
- На H=5000/C=16 `after` лучше по latency. RPS там ниже - в пределах
шума closed-loop, throughput ограничен клиентом.
- На H=100 разница в пределах шума — сортировка короткого списка дёшева.
- Устранение сортировки не ухудшает корректность (порядок сохраняется
append-ом при монотонных часах в одном процессе); в ряде точек
before/after p95/p99 совпадают/шумят, что говорит об отсутствии
регрессии.
- errors = 0 во всех точках обоих прогонов.
## Ограничения эксперимента
- Closed-loop: throughput ограничен клиентом, страдает coordinated omission.
- Один Docker-host: HA/durability не проверяются и не заявляются.
- Абсолютные p95/p99 не переносимы на другой ноутбук.
- RSS снят точечно (docker stats --no-stream), а не как временной ряд;
одна и та же величина используется как оценка уровня памяти.
- Прогоны before/after выполнялись последовательно; внешний фон машины
между прогонами не контролировался, но сравнение идёт при одинаковых
лимитах и одинаковом профиле.
## Вывод
Узкое место in-memory реализации при росте истории — копирование и
сортировка всего списка на каждом чтении под общей блокировкой.
Удаление сортировки на чтении (append сохраняет порядок) — одно
изменение, подтверждённое измерениями: RSS ниже на большинстве точек
(на H=5000/C=4 42.15 против 42.75 МБ, на H=5000/C=16 43.08 против
44.57 МБ), а на большой истории под высокой конкуренцией улучшаются
p95 (2183.0 → 2116.6 мс) и p99 (2188.8 → 2124.5 мс) без потери
корректности. Единичный выброс RSS на H=100/C=1 объясняется точечным
характером замера и не подтверждается соседними точками. Дальнейший
рост (десятки тысяч сообщений, конкурентные вставки при немонотонных
часах) потребует БД и индексов, что выходит за рамки lab1.
```
Ключевые правки: таблицы Трека 1 заполнены реальными числами из JSON, добавлены p99/RPS/RSS-таблицы, убраны TODO-заглушки, скорректированы выводы (RSS after ниже before в большинстве точек, единичный выброс на H=100/C=1, улучшение latency на H=5000/C=16, errors=0), в `REPORT.md` раздел Трека 1 приведён в соответствие с summary.
+139
View File
@@ -0,0 +1,139 @@
# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса
Данные: 250002 сообщений, неравномерные чаты. Запрос: `SELECT ... FROM messages WHERE chat_id = <id> AND (created_at, id) > (<середина>, …) ORDER BY created_at, id LIMIT 51`.
Большой чат ~151818, маленький чат ~1.
## 1. Без индекса, большой чат
`Parallel Seq Scan` по всей таблице, затем `Sort`.
```
Limit (cost=6593.26..6599.20 rows=51 width=70) (actual time=10.966..12.790 rows=51.00 loops=1)
Buffers: shared hit=3210
-> Gather Merge (cost=6593.26..11539.13 rows=42466 width=70) (actual time=10.964..12.782 rows=51.00 loops=1)
Workers Planned: 2
Workers Launched: 2
Buffers: shared hit=3210
-> Sort (cost=5593.24..5637.48 rows=17694 width=70) (actual time=7.715..7.718 rows=39.00 loops=3)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 36kB
Buffers: shared hit=3210
Worker 0: Sort Method: top-N heapsort Memory: 36kB
Worker 1: Sort Method: top-N heapsort Memory: 36kB
-> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=17694 width=70) (actual time=3.301..5.795 rows=26666.67 loops=3)
Filter: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
Rows Removed by Filter: 56667
Buffers: shared hit=3180
Planning:
Buffers: shared hit=88
Planning Time: 0.571 ms
Execution Time: 12.922 ms
```
## 2. Без индекса, маленький чат
Тот же `Parallel Seq Scan` по всей таблице.
```
Limit (cost=6003.04..6003.05 rows=1 width=70) (actual time=7.881..10.103 rows=1.00 loops=1)
Buffers: shared hit=3186
-> Sort (cost=6003.04..6003.05 rows=1 width=70) (actual time=7.879..10.099 rows=1.00 loops=1)
Sort Key: created_at, id
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=3186
-> Gather (cost=1000.00..6003.03 rows=1 width=70) (actual time=7.667..10.055 rows=1.00 loops=1)
Workers Planned: 2
Workers Launched: 2
Buffers: shared hit=3180
-> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=1 width=70) (actual time=3.431..4.199 rows=0.33 loops=3)
Filter: ((chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
Rows Removed by Filter: 83334
Buffers: shared hit=3180
Planning:
Buffers: shared hit=88
Planning Time: 0.662 ms
Execution Time: 10.192 ms
```
## 3. Индекс (chat_id), большой чат
`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`.
```
Limit (cost=6594.20..6600.14 rows=51 width=70) (actual time=9.705..11.642 rows=51.00 loops=1)
Buffers: shared hit=3210
-> Gather Merge (cost=6594.20..11547.87 rows=42533 width=70) (actual time=9.688..11.619 rows=51.00 loops=1)
Workers Planned: 2
Workers Launched: 2
Buffers: shared hit=3210
-> Sort (cost=5594.17..5638.48 rows=17722 width=70) (actual time=6.872..6.874 rows=37.33 loops=3)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 36kB
Buffers: shared hit=3210
Worker 0: Sort Method: top-N heapsort Memory: 36kB
Worker 1: Sort Method: top-N heapsort Memory: 37kB
-> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=17722 width=70) (actual time=2.667..5.224 rows=26666.67 loops=3)
Filter: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
Rows Removed by Filter: 56667
Buffers: shared hit=3180
Planning:
Buffers: shared hit=100
Planning Time: 0.552 ms
Execution Time: 11.745 ms
```
## 4. Индекс (chat_id), маленький чат
`Index Scan using idx_messages_chat_only` + `Sort`.
```
Limit (cost=32.15..32.16 rows=4 width=70) (actual time=0.055..0.056 rows=1.00 loops=1)
Buffers: shared hit=10
-> Sort (cost=32.15..32.16 rows=4 width=70) (actual time=0.054..0.055 rows=1.00 loops=1)
Sort Key: created_at, id
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=10
-> Index Scan using idx_messages_chat_only on messages (cost=0.42..32.11 rows=4 width=70) (actual time=0.022..0.023 rows=1.00 loops=1)
Index Cond: (chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid)
Filter: (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))
Index Searches: 1
Buffers: shared hit=4
Planning:
Buffers: shared hit=100
Planning Time: 0.642 ms
Execution Time: 0.112 ms
```
## 5. Полный индекс (chat_id, created_at, id), большой чат
`Index Scan using idx_messages_chat_created`, **без `Sort`**.
```
Limit (cost=0.42..17.05 rows=51 width=70) (actual time=0.093..0.150 rows=51.00 loops=1)
Buffers: shared hit=53 read=1
-> Index Scan using idx_messages_chat_created on messages (cost=0.42..13689.96 rows=41992 width=70) (actual time=0.092..0.146 rows=51.00 loops=1)
Index Cond: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
Index Searches: 1
Buffers: shared hit=53 read=1
Planning:
Buffers: shared hit=111
Planning Time: 0.751 ms
Execution Time: 0.208 ms
```
## 6. Полный индекс (chat_id, created_at, id), маленький чат
`Index Scan` без `Sort`.
```
Limit (cost=0.42..8.44 rows=1 width=70) (actual time=0.028..0.029 rows=1.00 loops=1)
Buffers: shared hit=4
-> Index Scan using idx_messages_chat_created on messages (cost=0.42..8.44 rows=1 width=70) (actual time=0.027..0.028 rows=1.00 loops=1)
Index Cond: ((chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
Index Searches: 1
Buffers: shared hit=4
Planning:
Buffers: shared hit=111
Planning Time: 0.846 ms
Execution Time: 0.105 ms
```
+8
View File
@@ -0,0 +1,8 @@
Track 2 restore
dump_time_s=1.701
dump_size_bytes=7783813
restore_time_s=1.623
count_original=280002
count_restored=280002
dist_match=true
random100_match=true
+4
View File
@@ -0,0 +1,4 @@
Track 2 RPO
marker_message_id=9c0393a4-4fbd-4c79-8229-1fdd065eb7d8
present_in_restore=0
expected=0
+20
View File
@@ -0,0 +1,20 @@
[
{
"variant": "no_index",
"insert_10000_s": 0.873,
"indexes_size": "10 MB",
"total_size": "36 MB"
},
{
"variant": "chat_only",
"insert_10000_s": 0.888,
"indexes_size": "12 MB",
"total_size": "39 MB"
},
{
"variant": "full",
"insert_10000_s": 0.805,
"indexes_size": "26 MB",
"total_size": "54 MB"
}
]
@@ -0,0 +1,13 @@
apiVersion: 1
contactPoints:
- orgId: 1
name: messenger-local
receivers:
- uid: messenger-local-webhook
type: webhook
disableResolveMessage: false
settings:
# Доставка наружу не требуется; локальный no-op endpoint.
url: http://127.0.0.1:9/messenger-alerts
httpMethod: POST
@@ -0,0 +1,9 @@
apiVersion: 1
policies:
- orgId: 1
receiver: messenger-local
group_by: ['alertname']
group_wait: 10s
group_interval: 1m
repeat_interval: 4h
@@ -0,0 +1,109 @@
apiVersion: 1
groups:
- orgId: 1
name: messenger-alerts
folder: Messenger
interval: 1m
rules:
- uid: messenger-high-error-rate
title: MessengerHighErrorRate
condition: C
data:
- refId: A
relativeTimeRange:
from: 60
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: sum(rate(http_requests_total{http_status_class="5xx"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])), 1e-9)
instant: true
intervalMs: 1000
maxDataPoints: 43200
range: false
refId: A
- refId: C
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: [0.05]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
for: 30s
annotations:
summary: "API error rate > 5% over 1m"
description: "5xx ratio exceeds SLO 5% for 30 seconds."
runbook: "infra/runbooks/high-error-rate.md"
labels:
severity: warning
noDataState: NoData
execErrState: Error
isPaused: false
- uid: messenger-jobs-backlog-growing
title: MessengerJobsBacklogGrowing
condition: C
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: sum(increase(jobs_processed_total{outcome="retry"}[10m])) - sum(increase(jobs_processed_total{outcome="success"}[10m]))
instant: true
intervalMs: 1000
maxDataPoints: 43200
range: false
refId: A
- refId: C
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: [5]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
for: 5m
annotations:
summary: "Retries outpace successes"
description: "Worker retry count exceeds successes by >5 for 5m."
runbook: "infra/runbooks/queue-lag.md"
labels:
severity: warning
noDataState: NoData
execErrState: Error
isPaused: false
@@ -0,0 +1,84 @@
{
"uid": "messenger-lab6",
"title": "Messenger — lab6 observability",
"schemaVersion": 39,
"version": 2,
"editable": true,
"timezone": "browser",
"time": {"from": "now-30m", "to": "now"},
"templating": {
"list": [
{
"name": "service",
"type": "query",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"query": "label_values(up, job)",
"current": {"text": "otel-collector", "value": "otel-collector"}
}
]
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "API RPS (total)",
"gridPos": {"x": 0, "y": 0, "w": 8, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps", "refId": "A"}
]
},
{
"id": 2,
"type": "timeseries",
"title": "API error ratio (5xx / total)",
"gridPos": {"x": 8, "y": 0, "w": 8, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio", "refId": "A"}
]
},
{
"id": 3,
"type": "timeseries",
"title": "HTTP latency p50/p95/p99 (s)",
"gridPos": {"x": 16, "y": 0, "w": 8, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Jobs processed by outcome",
"gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A"}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Job duration p95 (s) by outcome",
"gridPos": {"x": 12, "y": 8, "w": 12, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}", "refId": "A"}
]
},
{
"id": 6,
"type": "logs",
"title": "API + worker logs",
"gridPos": {"x": 0, "y": 16, "w": 24, "h": 10},
"datasource": {"type": "loki", "uid": "loki"},
"targets": [
{"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}", "refId": "A"}
]
}
]
}
@@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
options:
path: /etc/grafana/provisioning/dashboards/json
foldersFromFilesStructure: false
@@ -0,0 +1,37 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherRegex: 'trace_id=([a-f0-9]+)'
datasourceUid: tempo
url: "$${__value.raw}"
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
tags:
- key: service.name
value: service_name
filterByTraceID: true
+42
View File
@@ -0,0 +1,42 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
retention_period: 168h
reject_old_samples: true
reject_old_samples_max_age: 168h
allow_structured_metadata: true
compactor:
working_directory: /loki/compactor
delete_request_store: filesystem
retention_enabled: true
analytics:
reporting_enabled: false
+48
View File
@@ -0,0 +1,48 @@
# vhost для nginx: монтируется в /etc/nginx/conf.d/default.conf.
# Главный nginx.conf образа сам содержит events/http/include conf.d/*.conf.
# DNS Docker: 127.0.0.11 — встроенный резолвер, отдаёт адреса сервисов
# compose-сети. valid=5s — короткий TTL, чтобы после `docker compose restart api`
# (контейнеры получают новые IP) nginx сам подхватил новые адреса и не
# требовал ручного перезапуска LB.
resolver 127.0.0.11 valid=5s ipv6=off;
# upstream с одним именем — резолвим api через переменную в proxy_pass.
# Без `server api:8080;` в upstream, иначе nginx закэширует IP на старте.
# Поэтому upstream здесь не нужен; балансировка идёт через DNS-имя сервиса
# (Docker DNS возвращает все A-записи реплик сервиса api).
# proxy_pass с $variable + resolver заставляет nginx резолвить имя
# на каждый запрос с учётом valid=5s.
server {
listen 80 default_server;
server_name _;
# Не ограничивать допустимый размер вложения приложением (MAX_UPLOAD_BYTES
# = 10 MiB); nginx по умолчанию отвечает 413 HTML. Значение "10m" не
# меньше 10485760 байт, поэтому валидные загрузки до 10 MiB проходят.
client_max_body_size 10m;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
location / {
# Переменная в proxy_pass: nginx резолвит api:8080 через resolver
# на каждый запрос, учитывая valid=5s. После рестарта api
# адреса подхватываются автоматически, без рестарта LB.
# Без upstream{} и без `server api:8080;` — иначе IP кэшируется.
set $api_upstream "api:8080";
proxy_pass http://$api_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
location = /lb/health {
return 200 "ok\n";
add_header Content-Type text/plain;
}
}
+80
View File
@@ -0,0 +1,80 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
grpc:
endpoint: 0.0.0.0:4317
processors:
# Level-4: ограничение RAM и batch, чтобы не расти бесконечно при
# недоступном backend и уменьшить объём записи.
memory_limiter:
check_interval: 2s
limit_mib: 192
spike_limit_mib: 64
batch:
timeout: 5s
send_batch_size: 1024
send_batch_max_size: 2048
resource:
attributes:
- key: deployment.environment.name
value: lab
action: upsert
exporters:
# Явно ограничиваем retry и время ожидания, чтобы при недоступном
# бэкенде коллектор не накапливал данные и не падал.
otlphttp/tempo:
endpoint: http://tempo:4318
tls:
insecure: true
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
sending_queue:
enabled: true
num_consumers: 4
queue_size: 1000
otlphttp/loki:
endpoint: http://loki:3100/otlp
tls:
insecure: true
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
sending_queue:
enabled: true
num_consumers: 4
queue_size: 1000
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
send_timestamps: true
metric_expiration: 5m
service:
telemetry:
logs:
level: warn
metrics:
address: 0.0.0.0:8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, resource, batch]
exporters: [otlphttp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, resource, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, resource, batch]
exporters: [otlphttp/loki]
+12
View File
@@ -0,0 +1,12 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/rules/*.yaml
scrape_configs:
- job_name: otel-collector
scrape_interval: 5s
static_configs:
- targets: ["otel-collector:8889"]
+44
View File
@@ -0,0 +1,44 @@
groups:
- name: messenger-alerts
interval: 30s
rules:
- alert: MessengerHighErrorRate
expr: |
sum(rate(http_requests_total{http_status_class="5xx"}[1m]))
/
sum(rate(http_requests_total[1m]))
> 0.05
for: 30s
labels:
severity: warning
annotations:
summary: "API error rate > 5% over 1m"
description: "5xx ratio exceeds SLO 5% for 30 seconds."
runbook: "infra/runbooks/high-error-rate.md"
- alert: MessengerJobsBacklogGrowing
expr: |
sum(increase(jobs_processed_total{outcome="retry"}[10m]))
>
sum(increase(jobs_processed_total{outcome="success"}[10m]))
+ 5
for: 5m
labels:
severity: warning
annotations:
summary: "Retries outpace successes"
description: "Worker retry count exceeds successes for 5m."
runbook: "infra/runbooks/queue-lag.md"
- alert: MessengerUploadLatencyHigh
expr: |
histogram_quantile(0.95,
sum by (le, http_route)
(rate(http_request_duration_seconds_bucket{http_route=~"/api/v1/chats/.*/attachments"}[5m]))
) > 2
for: 3m
labels:
severity: warning
annotations:
summary: "p95 upload latency > 2s"
runbook: "infra/runbooks/slow-upload.md"
+19
View File
@@ -0,0 +1,19 @@
# Alert: MessengerHighErrorRate
## Симптомы
- 5xx rate > 5% в течение 2 минут.
- Панель «API error ratio» в Grafana.
## Диагностика (по шагам)
1. Открыть Grafana → Dashboard «Messenger» → p95 latency.
2. Открыть Explore → Loki: `{service_name="messenger-api"} |= "event=unhandled_exception"`.
3. Найти в логах `request_id`, затем в Tempo trace с этим span или по
trace_id из поля `trace_id`.
4. Смотрим какие зависимость отвечает: postgres/redis/s3. Соответствующие
ошибки пишутся как `event=db_unavailable`, `event=redis_unavailable`,
`event=s3_unavailable`.
## Действие
- Если виноват backend — перезапустить (`docker compose restart <service>`).
- Если ошибка идёт от кода — rollback через `docker compose up -d --build`
предыдущего тега.
+17
View File
@@ -0,0 +1,17 @@
# Alert: MessengerJobsBacklogGrowing
## Симптомы
- `jobs_processed_total{outcome="retry"}` > `success` + 5 за 10 минут.
## Диагностика
1. Проверить `docker compose ps worker` — процесс жив?
2. `docker compose exec rabbitmq rabbitmqctl list_queues name messages consumers`
3. Открыть Grafana → «Jobs processed by outcome» и «Job duration p95».
4. Открыть Tempo, найти span `job.consume` с ошибкой (status=ERROR).
## Действие
- Если worker упал — `docker compose restart worker`.
- Если очередь переполнена по `retry_exhausted` — проверить S3 (`minio`
контейнер healthy), затем `docker compose restart s3_cleanup`.
- При долгом backlog включить alert `for: 10m` и снизить `MAX_ATTEMPTS` в `.env`
для быстрого drain.
+15
View File
@@ -0,0 +1,15 @@
# Alert: MessengerUploadLatencyHigh
## Симптомы
- p95 длительности POST `/api/v1/chats/{id}/attachments` > 2s.
## Диагностика
1. Grafana: p95 по route `/api/v1/chats/{id}/attachments`.
2. Tempo: trace `POST /api/v1/chats/{id}/attachments` — найти самый долгий
child span (`put_object` в S3 или `INSERT` в postgres).
3. Loki: `{service_name="messenger-api"} |= "event=http" |~ "attachments"`.
## Действие
- Если виноват S3 — проверить `docker compose ps minio`.
- Если БД — увеличить `max_size` пула или `statement_timeout`.
- Если телеметрия — переключить sampling (см. TRACK1).
+34
View File
@@ -0,0 +1,34 @@
server:
http_listen_port: 3200
grpc_listen_port: 9095
log_level: warn
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
grpc:
endpoint: 0.0.0.0:4317
ingester:
trace_idle_period: 10s
max_block_bytes: 1048576
max_block_duration: 5m
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
local:
path: /var/tempo/traces
wal:
path: /var/tempo/wal
metrics_generator:
storage:
path: /var/tempo/generator/wal
+13
View File
@@ -0,0 +1,13 @@
fastapi==0.115.0
uvicorn[standard]==0.32.0
psycopg[binary]==3.2.3
psycopg_pool==3.2.3
redis==5.2.0
boto3==1.35.74
Pillow==11.0.0
pika==1.3.2
opentelemetry-api==1.31.0
opentelemetry-sdk==1.31.0
opentelemetry-exporter-otlp-proto-http==1.31.0
opentelemetry-instrumentation-fastapi==0.52b0
opentelemetry-instrumentation-requests==0.52b0
+26
View File
@@ -0,0 +1,26 @@
@echo off
REM Windows-эквивалент scripts/restart.sh для lab4
REM Перезапускаем LB, обе реплики API, PostgreSQL, Redis, MinIO и s3_cleanup (тома сохраняются)
setlocal enableextensions
docker compose --env-file .env restart lb api postgres redis minio s3_cleanup >nul 2>&1
if errorlevel 1 (
docker restart mtuci-lb >nul 2>&1
docker restart mtuci-postgres >nul 2>&1
docker restart mtuci-redis >nul 2>&1
docker restart mtuci-minio >nul 2>&1
for /f "tokens=*" %%c in ('docker ps -a --filter "name=mtuci2026lab4_api" -q') do (
docker restart %%c >nul 2>&1
)
)
REM Ждём readiness до 90 секунд
set /a i=0
:wait_loop
if %i% geq 90 exit /b 1
curl -fsS http://localhost:8080/health/ready >nul 2>&1
if not errorlevel 1 exit /b 0
set /a i+=1
timeout /t 1 /nobreak >nul
goto wait_loop
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
set -eu
# Перезапускаем LB, обе реплики API, PostgreSQL, Redis, MinIO и s3_cleanup (тома сохраняются)
docker compose --env-file .env restart lb api postgres redis minio s3_cleanup >/dev/null 2>&1 || {
docker restart mtuci-lb >/dev/null 2>&1 || true
docker restart mtuci-postgres >/dev/null 2>&1 || true
docker restart mtuci-redis >/dev/null 2>&1 || true
docker restart mtuci-minio >/dev/null 2>&1 || true
for c in $(docker ps -a --filter "name=mtuci2026lab4_api" -q); do
docker restart "$c" >/dev/null 2>&1 || true
done
}
# Ждём readiness
i=0
while [ $i -lt 90 ]; do
if curl -fsS http://localhost:8080/health/ready >/dev/null 2>&1; then
exit 0
fi
i=$((i+1)); sleep 1
done
exit 1
+47
View File
@@ -0,0 +1,47 @@
#!/bin/sh
# Поочерёдное обновление API-реплик под нагрузкой.
# Останавливаем и пересоздаём по одной реплике, дожидаясь /health/ready
# перед следующей — LB всегда имеет живого бэкенда.
set -eu
BASE_URL="${BASE_URL:-http://localhost:8080}"
ENV_FILE="${ENV_FILE:-.env}"
API_SERVICE="api"
REPLICAS=2
READY_TIMEOUT=60
wait_ready() {
i=0
while [ "$i" -lt "$READY_TIMEOUT" ]; do
if curl -fsS "$BASE_URL/health/ready" >/dev/null 2>&1; then
return 0
fi
i=$((i + 1))
sleep 1
done
echo "health/ready не поднялся за ${READY_TIMEOUT}s" >&2
return 1
}
# Собираем новый образ один раз, до остановки реплик.
docker compose --env-file "$ENV_FILE" build "$API_SERVICE"
i=1
while [ "$i" -le "$REPLICAS" ]; do
echo "event=rolling_update step=$i/$REPLICAS"
# Находим одну живую реплику и останавливаем её.
cid=$(docker ps --filter "label=com.docker.compose.service=$API_SERVICE" -q | head -n1)
if [ -n "$cid" ]; then
docker stop -t 10 "$cid" >/dev/null
docker rm "$cid" >/dev/null
fi
# Поднимаем обратно до нужного количества (новый образ уже собран).
docker compose --env-file "$ENV_FILE" up -d --no-deps --scale "$API_SERVICE=$REPLICAS" "$API_SERVICE" >/dev/null
wait_ready
i=$((i + 1))
done
echo "event=rolling_update_done"
+39
View File
@@ -0,0 +1,39 @@
@echo off
REM Windows-эквивалент scripts/stop-one.sh для lab3.
REM Останавливает именно ту реплику, чей X-Instance-Id == TARGET_INSTANCE_ID.
REM DB/LB/volumes не трогаем; восстановление — docker compose up -d api.
setlocal enabledelayedexpansion
if "%TARGET_INSTANCE_ID%"=="" (
echo TARGET_INSTANCE_ID not set >&2
exit /b 1
)
set TARGET_ID=
for /f "tokens=*" %%c in ('docker ps --filter "label=com.docker.compose.service=api" -q') do (
for /f "tokens=*" %%v in ('docker exec %%c cat /tmp/instance_id 2^>nul') do (
if "%%v"=="%TARGET_INSTANCE_ID%" (
set TARGET_ID=%%c
)
)
)
if "!TARGET_ID!"=="" (
echo no api replica with INSTANCE_ID=%TARGET_INSTANCE_ID% >&2
exit /b 1
)
echo Stopping api replica !TARGET_ID! (INSTANCE_ID=%TARGET_INSTANCE_ID%)
docker stop -t 3 !TARGET_ID! >nul
REM Ждём, пока контейнер реально уйдёт из Up (максимум ~5 секунд).
set /a i=0
:wait_loop
if %i% geq 50 exit /b 0
for /f "tokens=*" %%c in ('docker ps --filter "id=!TARGET_ID!" --format "{{.ID}}"') do (
timeout /t 1 /nobreak >nul
set /a i+=1
goto wait_loop
)
exit /b 0
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
TARGET=""
for c in $(docker ps --filter "label=com.docker.compose.service=api" -q); do
id=$(docker exec "$c" cat /tmp/instance_id 2>/dev/null || true)
if [ "$id" = "$TARGET_INSTANCE_ID" ]; then
TARGET="$c"
break
fi
done
if [ -z "$TARGET" ]; then
echo "no api replica with INSTANCE_ID=$TARGET_INSTANCE_ID" >&2
exit 1
fi
docker stop -t 3 "$TARGET" >/dev/null
i=0
while [ $i -lt 50 ]; do
if ! docker ps --filter "id=$TARGET" --format "{{.ID}}" | grep -q .; then
exit 0
fi
i=$((i+1)); sleep 0.1
done
exit 0
View File
View File
+246
View File
@@ -0,0 +1,246 @@
"""Вложения (lab5/lab6): upload → 202 queued, async worker, download вариантов.
lab6: при создании outbox-строки кладём в payload W3C traceparent/tracestate
текущего HTTP-запроса. Ошибка инжекта не должна блокировать upload.
"""
import hashlib
import io
import os
import unicodedata
from datetime import datetime, timezone
from typing import Optional
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import Response
from PIL import Image, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from botocore.exceptions import ClientError
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from ..models.schemas import Attachment, User
from ..storage import s3
from ..storage.postgres import storage
from .security import current_user
router = APIRouter(prefix="/api/v1", tags=["attachments"])
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
PIPELINE_VERSION = 1
_propagator = TraceContextTextMapPropagator()
def _require_member(chat_id: UUID, user_id: UUID):
chat = storage.get_chat(chat_id)
if chat is None or user_id not in chat.member_ids:
raise HTTPException(status_code=404, detail="not_found")
return chat
def _validate_filename(name: str) -> str:
if not name or len(name) > 255:
raise HTTPException(status_code=400, detail="invalid_request")
if name.strip() != name or name.strip() == "":
raise HTTPException(status_code=400, detail="invalid_request")
for ch in name:
if ch in ("/", "\\", "\x00") or unicodedata.category(ch).startswith("C"):
raise HTTPException(status_code=400, detail="invalid_request")
return name
def _detect_content_type(data: bytes) -> Optional[str]:
"""MIME по содержимому, не по заголовку."""
if data.startswith(b"\x89PNG\r\n\x1a\n"):
return "image/png"
if data.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
return None
def _validate_image(data: bytes) -> str:
"""MIME по содержимому + лимит пикселей + полная проверка декодирования."""
ctype = _detect_content_type(data)
if ctype is None:
raise HTTPException(status_code=415, detail="unsupported_media_type")
try:
img = Image.open(io.BytesIO(data))
width, height = img.size
if width <= 0 or height <= 0:
raise HTTPException(status_code=415, detail="unsupported_media_type")
if width * height > MAX_IMAGE_PIXELS:
raise HTTPException(status_code=413, detail="payload_too_large")
img.load()
except HTTPException:
raise
except Image.DecompressionBombError:
raise HTTPException(status_code=413, detail="payload_too_large")
except Exception:
raise HTTPException(status_code=415, detail="unsupported_media_type")
return ctype
@router.post(
path="/chats/{chat_id}/attachments",
summary="Принять один PNG/JPEG, до 10 MiB (lab5: 202 queued)",
response_model=Attachment,
status_code=202,
)
async def upload_attachment(
chat_id: UUID,
request: Request,
user: User = Depends(current_user),
filename: str = Query(...),
) -> Attachment:
_require_member(chat_id, user.id)
safe_name = _validate_filename(filename)
cl = request.headers.get("content-length")
if cl is not None:
try:
if int(cl) > MAX_UPLOAD_BYTES:
raise HTTPException(status_code=413, detail="payload_too_large")
except ValueError:
raise HTTPException(status_code=400, detail="invalid_request")
chunks: list[bytes] = []
total = 0
async for chunk in request.stream():
if not chunk:
continue
total += len(chunk)
if total > MAX_UPLOAD_BYTES:
raise HTTPException(status_code=413, detail="payload_too_large")
chunks.append(chunk)
body = b"".join(chunks)
if not body:
raise HTTPException(status_code=400, detail="invalid_request")
content_type = await run_in_threadpool(_validate_image, body)
sha256 = hashlib.sha256(body).hexdigest()
attachment_id = uuid4()
staging_key = s3.staging_key(chat_id, attachment_id)
# 1) Durable staging в S3 (приватный prefix).
try:
await run_in_threadpool(s3.put_bytes, staging_key, body, content_type)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=503, detail="unavailable") from exc
now = datetime.now(timezone.utc)
att = Attachment(
id=attachment_id,
chat_id=chat_id,
owner_id=user.id,
filename=safe_name,
content_type=content_type,
size_bytes=len(body),
sha256=sha256,
status="queued",
variants=[],
error_code=None,
created_at=now,
updated_at=now,
)
job_id = uuid4()
# lab6: снимок W3C trace context активного HTTP span.
trace_carrier: dict[str, str] = {}
try:
_propagator.inject(trace_carrier)
except Exception:
trace_carrier = {}
outbox_payload = {
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": PIPELINE_VERSION,
"created_at": now.isoformat().replace("+00:00", "Z"),
}
if trace_carrier.get("traceparent"):
outbox_payload["traceparent"] = trace_carrier["traceparent"]
if trace_carrier.get("tracestate"):
outbox_payload["tracestate"] = trace_carrier["tracestate"]
# 2) INSERT attachment + outbox в ОДНОЙ транзакции.
# При сбое — компенсация: удаляем staging.
try:
await run_in_threadpool(
storage.create_attachment_queued, att, staging_key, outbox_payload
)
except Exception:
try:
await run_in_threadpool(s3.delete_object, staging_key)
except Exception:
pass
raise HTTPException(status_code=503, detail="unavailable")
return att
@router.get(
path="/attachments/{attachment_id}",
summary="Метаданные вложения, доступ участникам чата",
response_model=Attachment,
)
def get_attachment(
attachment_id: UUID,
user: User = Depends(current_user),
) -> Attachment:
pair = storage.get_attachment_with_key(attachment_id)
if pair is None:
raise HTTPException(status_code=404, detail="not_found")
att, _key = pair
_require_member(att.chat_id, user.id)
return att
@router.get(
path="/attachments/{attachment_id}/content",
summary="Скачать байты (original или variant) через авторизованный API",
)
def download_attachment(
attachment_id: UUID,
user: User = Depends(current_user),
variant: str = Query("original"),
):
if variant not in ("original", "thumbnail", "crop", "optimized"):
raise HTTPException(status_code=400, detail="invalid_request")
pair = storage.get_attachment_with_key(attachment_id)
if pair is None:
raise HTTPException(status_code=404, detail="not_found")
att, object_key = pair
_require_member(att.chat_id, user.id)
if att.status != "ready":
raise HTTPException(status_code=409, detail="conflict")
if variant == "original":
key = object_key
ctype_expected = att.content_type
else:
vs = [v for v in att.variants if v.name == variant]
if not vs:
raise HTTPException(status_code=404, detail="not_found")
v = vs[0]
key = s3.variant_key(att.chat_id, att.id, variant)
ctype_expected = v.content_type
try:
data, ctype = s3.get_object(key)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
if code in ("404", "NoSuchKey", "NotFound"):
raise HTTPException(status_code=404, detail="not_found")
raise HTTPException(status_code=503, detail="unavailable") from exc
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=503, detail="unavailable") from exc
return Response(
content=data,
media_type=ctype or ctype_expected,
)
+81
View File
@@ -0,0 +1,81 @@
import base64
import binascii
import os
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from ..models.schemas import Session
from ..storage.postgres import storage
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
DEFAULT_TTL_SECONDS = 3600
def _ttl_seconds() -> int:
try:
return int(os.environ.get("SESSION_TTL_SECONDS", DEFAULT_TTL_SECONDS))
except ValueError:
return DEFAULT_TTL_SECONDS
def _parse_basic_header(request: Request) -> tuple[str, str] | None:
"""Ручной разбор Basic: base64 -> UTF-8 -> split по первому ':'.
HTTPBasic из Starlette декодирует latin-1 и ломает кириллицу.
Здесь пароль читается как UTF-8 (RFC 7617 разрешает UTF-8 charset).
Username остаётся ASCII по контракту.
"""
raw = request.headers.get("Authorization", "")
if not raw.lower().startswith("basic "):
return None
try:
decoded = base64.b64decode(raw[6:].strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError):
return None
if ":" not in decoded:
return None
username, password = decoded.split(":", 1)
if not username or not password:
return None
try:
username.encode("ascii")
except UnicodeEncodeError:
return None
return username, password
@router.post(
path="/sessions",
summary="Вход: Basic username:password; выдаёт сессию",
response_model=Session,
status_code=201,
)
def create_session(request: Request) -> Session:
parsed = _parse_basic_header(request)
if parsed is None:
raise HTTPException(status_code=401, detail="unauthorized")
username, password = parsed
user = storage.verify_credentials(username, password)
if user is None:
raise HTTPException(status_code=401, detail="unauthorized")
token, expires_at = storage.create_session(user.id, _ttl_seconds())
return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user)
@router.delete(
path="/sessions/current",
summary="Отозвать текущую сессию",
status_code=204,
)
def revoke_session(request: Request) -> None:
auth = request.headers.get("Authorization", "")
if not auth.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="unauthorized")
token = auth[7:].strip()
if not token or storage.get_session_user(token) is None:
raise HTTPException(status_code=401, detail="unauthorized")
storage.revoke_session(token)
return None
+94
View File
@@ -0,0 +1,94 @@
import base64
import json
from datetime import datetime
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query
from ..models.schemas import Chat, ChatPage, CreateChat, User
from ..storage.postgres import storage
from .security import current_user
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
def _encode_cursor(user_id: UUID, chat: Chat) -> str:
"""Простой курсор: user_id + id + created_at, base64."""
data = {"user_id": str(user_id), "id": str(chat.id), "created_at": chat.created_at.isoformat()}
return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
def _decode_cursor(user_id: UUID, cursor: str) -> tuple[datetime, UUID]:
"""Курсор привязан к пользователю. Любая порча → 400, не 500."""
try:
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
data = json.loads(raw)
except Exception:
raise HTTPException(status_code=400, detail="invalid_request")
if not isinstance(data, dict):
raise HTTPException(status_code=400, detail="invalid_request")
if data.get("user_id") != str(user_id):
raise HTTPException(status_code=400, detail="invalid_request")
try:
created_at = datetime.fromisoformat(data["created_at"])
chat_id = UUID(data["id"])
except (KeyError, ValueError, TypeError):
raise HTTPException(status_code=400, detail="invalid_request")
return created_at, chat_id
@router.post(
path="",
summary="Создать чат",
response_model=Chat,
status_code=201,
)
def create_chat(
payload: CreateChat,
user: User = Depends(current_user),
) -> Chat:
"""Вызывающий должен входить в member_ids"""
user_id = user.id
if user_id not in payload.member_ids:
raise HTTPException(status_code=400, detail="invalid_request")
if len(set(payload.member_ids)) != len(payload.member_ids):
raise HTTPException(status_code=400, detail="invalid_request")
try:
return storage.create_chat(payload.title, list(payload.member_ids))
except ValueError:
raise HTTPException(status_code=400, detail="invalid_request")
@router.get(
path="",
summary="Получить список чатов текущего пользователя",
response_model=ChatPage,
)
def list_chats(
user: User = Depends(current_user),
limit: int = Query(50, ge=1, le=100),
cursor: str | None = None,
) -> ChatPage:
user_id = user.id
chats = storage.list_chats_for_user(user_id)
if cursor:
ccreated, cid = _decode_cursor(user_id, cursor)
ccreated_iso = ccreated.isoformat()
chats = [c for c in chats if (c.created_at.isoformat(), str(c.id)) > (ccreated_iso, str(cid))]
page = chats[:limit]
next_cursor = _encode_cursor(user_id, page[-1]) if len(chats) > limit else None
return ChatPage(items=page, next_cursor=next_cursor)
@router.get(
path="/{chat_id}",
summary="Получить доступный чат по идентификатору",
response_model=Chat,
)
def get_chat(
chat_id: UUID,
user: User = Depends(current_user),
) -> Chat:
chat = storage.get_chat(chat_id)
if chat is None or user.id not in chat.member_ids:
raise HTTPException(status_code=404, detail="not_found")
return chat
+52
View File
@@ -0,0 +1,52 @@
from fastapi import APIRouter
from ..models.schemas import Health
from ..storage import db as _db
from ..storage import s3 as _s3
from ..storage.sessions import _redis_client, _store
router = APIRouter(tags=["health"])
@router.get(
path="/health/live",
summary="Проверка жизнеспособности процесса",
response_model=Health,
)
def liveness() -> Health:
"""Процесс жив"""
return Health(status="ok")
@router.get(
path="/health/ready",
summary="Проверка готовности принимать трафик",
response_model=Health,
)
def readiness() -> Health:
"""200, если PostgreSQL, Redis (в режиме redis) и S3 доступны.
При недоступности зависимости выбрасываем RuntimeError("*_unavailable"),
main.py конвертирует его в 503 с телом Error из contracts/openapi.json.
Все проверки — с короткими таймаутами, чтобы не зависать надолго.
"""
try:
with _db.connection() as conn, conn.cursor() as cur:
cur.execute("SELECT 1")
cur.fetchone()
except Exception as exc: # noqa: BLE001
# Пул уже проверяет живость соединения (check_connection) и короткий
# timeout=3.0; здесь лишь превращаем любую ошибку БД в 503, чтобы
# после рестарта PostgreSQL readiness быстро возвращался в 200.
raise RuntimeError("db_unavailable: %s" % type(exc).__name__) from exc
# Redis (в режиме redis)
if _store() == "redis":
_redis_client().ping()
# S3: HEAD несуществующего ключа. 404/None → S3 доступен.
# Короткий таймаут клиента гарантирует быстрый ответ, а ошибка
# преобразуется в RuntimeError("s3_unavailable") → 503.
try:
_s3.head_object("attachments/__health_probe__")
except Exception as exc: # noqa: BLE001
raise RuntimeError("s3_unavailable: %s" % type(exc).__name__) from exc
return Health(status="ok")
+141
View File
@@ -0,0 +1,141 @@
import base64
import hashlib
import json
from datetime import datetime
from uuid import UUID
from fastapi import APIRouter, Depends, Header, HTTPException, Query
from ..models.schemas import CreateMessage, Message, MessagePage, User
from ..storage.postgres import storage
from .security import current_user
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
# Контракт: Idempotency-Key ASCII [A-Za-z0-9._:-]{1,128}.
_IDEMPOTENCY_ALLOWED = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._:-")
def _require_member(chat_id: UUID, user_id: UUID):
chat = storage.get_chat(chat_id)
if chat is None or user_id not in chat.member_ids:
raise HTTPException(status_code=404, detail="not_found")
return chat
def _encode_cursor(chat_id: UUID, message: Message) -> str:
data = {"chat_id": str(chat_id), "id": str(message.id),
"created_at": message.created_at.isoformat()}
return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
def _decode_cursor(chat_id: UUID, cursor: str) -> tuple:
try:
data = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
if data.get("chat_id") != str(chat_id):
raise ValueError("cursor_chat_mismatch")
return datetime.fromisoformat(data["created_at"]), UUID(data["id"])
except Exception:
raise HTTPException(status_code=400, detail="invalid_request")
def _canonical_body_hash(payload: CreateMessage) -> str:
"""sha256 канонизированного JSON.
Поле attachment_ids присутствует в модели со значением по умолчанию [],
поэтому payload {"text":"x"} и {"text":"x","attachment_ids":[]} дают
одинаковый model_dump и одинаковый хэш — как требует контракт.
"""
data = payload.model_dump(mode="json")
canonical = json.dumps(data, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
@router.post(
path="/{chat_id}/messages",
summary="Отправить сообщение в чат",
response_model=Message,
status_code=201,
)
def create_message(
chat_id: UUID,
payload: CreateMessage,
user: User = Depends(current_user),
idempotency_key: str = Header(..., alias="Idempotency-Key"),
) -> Message:
# Строго ASCII, как требует контракт: isalnum() пропускает кириллицу.
if not (1 <= len(idempotency_key) <= 128):
raise HTTPException(status_code=400, detail="invalid_request")
if not all(c in _IDEMPOTENCY_ALLOWED for c in idempotency_key):
raise HTTPException(status_code=400, detail="invalid_request")
_require_member(chat_id, user.id)
text = payload.text or ""
attachment_ids = list(payload.attachment_ids)
# Контракт: text (непустой) ИЛИ attachment_ids (>=1).
if text.strip() == "" and not attachment_ids:
raise HTTPException(status_code=400, detail="invalid_request")
# Все вложения должны принадлежать этому чату.
if attachment_ids:
found = storage.list_attachments_by_ids(attachment_ids, chat_id)
if len(found) != len(attachment_ids):
raise HTTPException(status_code=404, detail="not_found")
for aid in attachment_ids:
att, _key = found[aid]
if att.status == "failed":
raise HTTPException(status_code=409, detail="conflict")
method = "POST"
path = "/api/v1/chats/%s/messages" % chat_id
body_hash = _canonical_body_hash(payload)
# Атомарная операция: INSERT ключа + INSERT сообщения + UPDATE ключа
# в ОДНОЙ транзакции. Сбой между ними невозможен: либо всё, либо ничего.
state, message, info = storage.add_message_idempotent(
chat_id=chat_id,
sender_id=user.id,
text=text,
attachment_ids=attachment_ids,
user_id=user.id,
method=method,
path=path,
key=idempotency_key,
body_hash=body_hash,
)
if state == "created":
return message
if state == "retry":
raise HTTPException(status_code=503, detail="unavailable")
# state == "replay"
if info["body_hash"] != body_hash:
raise HTTPException(status_code=409, detail="conflict")
if info["response_status"] == 0:
# Строка от старой версии кода: не зависаем, отдаём 503.
raise HTTPException(status_code=503, detail="unavailable")
return Message.model_validate(info["response_body"])
@router.get(
path="/{chat_id}/messages",
summary="Получить историю сообщений чата",
response_model=MessagePage,
)
def list_messages(
chat_id: UUID,
user: User = Depends(current_user),
limit: int = Query(50, ge=1, le=100),
cursor: str | None = None,
) -> MessagePage:
"""Keyset-пагинация в SQL: (created_at, id) по возрастанию"""
_require_member(chat_id, user.id)
after = _decode_cursor(chat_id, cursor) if cursor else None
rows = storage.list_messages_page(chat_id, limit + 1, after=after)
has_more = len(rows) > limit
items = rows[:limit]
next_cursor = _encode_cursor(chat_id, items[-1]) if has_more and items else None
return MessagePage(items=items, next_cursor=next_cursor)
+24
View File
@@ -0,0 +1,24 @@
from uuid import UUID
from fastapi import HTTPException, Security
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from ..models.schemas import User
from ..storage.postgres import storage
bearer_scheme = HTTPBearer(auto_error=False, scheme_name="bearerAuth")
def current_user(
credentials: HTTPAuthorizationCredentials | None = Security(bearer_scheme),
) -> User:
"""Достаёт Bearer-токен и возвращает владельца сессии.
Basic на бизнес-endpoint не принимается, X-User-Id игнорируется.
"""
if credentials is None or credentials.scheme.lower() != "bearer":
raise HTTPException(status_code=401, detail="unauthorized")
user = storage.get_session_user(credentials.credentials)
if user is None:
raise HTTPException(status_code=401, detail="unauthorized")
return user
+31
View File
@@ -0,0 +1,31 @@
from fastapi import APIRouter, Depends, HTTPException
from ..models.schemas import CreateUser, User
from ..storage.postgres import storage
from .security import current_user
router = APIRouter(prefix="/api/v1", tags=["users"])
@router.post(
path="/users",
summary="Создать пользователя",
response_model=User,
status_code=201,
)
def register_user(payload: CreateUser) -> User:
"""Публичная регистрация. Пробельные display_name/password отсекаются
валидаторами схемы; сюда попадают только валидные значения."""
try:
return storage.create_user(payload.username, payload.display_name, payload.password)
except ValueError:
raise HTTPException(status_code=409, detail="conflict")
@router.get(
path="/users/me",
summary="Получить текущего пользователя",
response_model=User,
)
def current_user_me(user: User = Depends(current_user)) -> User:
"""Текущий пользователь по Bearer-сессии."""
return user
+269
View File
@@ -0,0 +1,269 @@
import logging
import os
import signal
import sys
import time
import uuid
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.openapi.utils import get_openapi
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
import psycopg
import uvicorn
from opentelemetry import trace as _otel_trace
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from .api import attachments, auth, chats, health, messages, users
from .storage.db import init_pool, close_pool
from .telemetry import init_telemetry, INSTANCE_ID, meter, tracer
# Базовый логгер: реальный формат выставляет telemetry.install_log_format()
# при вызове init_telemetry(). Здесь — минимальный fallback для импорта.
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger("app")
@asynccontextmanager
async def lifespan(app: FastAPI):
init_pool()
logger.info(
"event=startup app_env=%s session_store=%s instance_id=%s",
os.environ.get("APP_ENV", "lab"),
os.environ.get("SESSION_STORE", "postgres"),
INSTANCE_ID,
)
yield
close_pool()
logger.info("event=shutdown_complete instance_id=%s", INSTANCE_ID)
app = FastAPI(
title="MTUCI Messenger — lab6",
version="1.6.0",
lifespan=lifespan,
swagger_ui_parameters={"persistAuthorization": True},
)
# OTel инициализация и FastAPI auto-instrumentation.
init_telemetry("messenger-api", "1.6.0")
# --- Метрики HTTP ---
_http_counter = meter("messenger").create_counter(
"http_requests_total",
description="HTTP requests count",
unit="1",
)
# Явные границы бакетов в СЕКУНДАХ, с разрешением в миллисекундном диапазоне
# и вокруг порога SLO 2s (upload p95 alert). Дефолтные экспоненциальные
# бакеты (0, 5, 10, 25, …) грубы: миллисекундные запросы попадали в один
# широкий bucket, а histogram_quantile интерполировал ложные секунды.
_HTTP_DURATION_BUCKETS = [
0.001, 0.0025, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05,
0.075, 0.1, 0.15, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0,
]
_http_hist = meter("messenger").create_histogram(
"http_request_duration_seconds",
description="HTTP request duration",
unit="s",
explicit_bucket_boundaries_advisory=_HTTP_DURATION_BUCKETS,
)
# Фильтр: убираем реальные UUID из маршрута → route template.
import re as _re
_ROUTE_UUID_RE = _re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
def _route_template(path: str) -> str:
return _ROUTE_UUID_RE.sub("{id}", path)
def custom_openapi():
if app.openapi_schema:
return app.openapi_schema
schema = get_openapi(title=app.title, version=app.version, routes=app.routes)
app.openapi_schema = schema
return schema
app.openapi = custom_openapi
# Отдельный tracer для событийного span в middleware. Он создаётся как child
# SERVER span, который создаёт OTel FastAPI instrumentation ниже.
# ВАЖНО: FastAPIInstrumentor.instrument_app(app) вызывается ПОСЛЕ объявления
# middleware, поэтому OTel ASGI middleware оказывается внешним: он извлекает
# W3C traceparent из входящих headers, продолжает внешний trace и создаёт
# SERVER span. Наш middleware выполняется внутри него, создаёт http.request
# как INTERNAL child, и логи/метрики берут корректный trace_id.
_http_tracer = tracer("app.http")
@app.middleware("http")
async def observability_middleware(request: Request, call_next):
"""X-Request-Id и X-Instance-Id во всех ответах + структурированный лог + метрики.
Лог event=http пишется ВНУТРИ активного span (`http.request`), он
child SERVER span от OTel auto-instrumentation, поэтому trace_id
корректно связывает лог с внешним trace.
"""
request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
request.state.request_id = request_id
started = time.monotonic()
route = _route_template(request.url.path)
# http.request — INTERNAL child текущего (SERVER) span, созданного OTel
# auto-instrumentation. Parent берётся из активного контекста: входящий
# traceparent уже извлечён ASGI middleware.
with _http_tracer.start_as_current_span(
"http.request",
attributes={
"http.method": request.method,
"http.route": route,
},
) as span:
response = await call_next(request)
duration_s = time.monotonic() - started
duration_ms = round(duration_s * 1000, 2)
status_class = "%dxx" % (response.status_code // 100)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
_http_counter.add(
1,
{"http.method": request.method, "http.route": route,
"http.status_class": status_class},
)
_http_hist.record(
duration_s,
{"http.method": request.method, "http.route": route,
"http.status_class": status_class},
)
span.set_attribute("http.status_class", status_class)
span.set_attribute("http.status_code", response.status_code)
# Лог ВНУТРИ span → record.trace_id валидный, Loki получает
# корреляцию с Tempo.
logger.info(
"event=http method=%s path=%s status=%s duration_ms=%s "
"request_id=%s instance_id=%s",
request.method, request.url.path, response.status_code,
duration_ms, request_id, INSTANCE_ID,
)
return response
# FastAPI auto-instrumentation ДОЛЖНА быть установлена ПОСЛЕ пользовательского
# @app.middleware("http"), чтобы оказаться внешним middleware и создать SERVER
# span с корректным parent из входного traceparent. Этот вызов также
# инструментирует httpx/requests client-вызовы внутри приложения.
FastAPIInstrumentor.instrument_app(app)
# --- Обработчики ошибок (без изменений) ---
_ERROR_CODES = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
503: "unavailable",
500: "internal_error",
}
def _unauthorized_challenge(path: str) -> str:
if path == "/api/v1/auth/sessions":
return 'Basic realm="messenger", charset="UTF-8"'
return 'Bearer realm="messenger"'
def _error_response(status_code: int, message: str, request_id: str,
path: str = "") -> JSONResponse:
code = _ERROR_CODES.get(status_code, "internal_error")
headers = {}
if status_code == 401:
headers["WWW-Authenticate"] = _unauthorized_challenge(path)
headers["X-Request-Id"] = request_id
headers["X-Instance-Id"] = INSTANCE_ID
return JSONResponse(
status_code=status_code,
content={"error": {"code": code, "message": message, "request_id": request_id}},
headers=headers,
)
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
message = exc.detail if isinstance(exc.detail, str) else "Ошибка"
return _error_response(exc.status_code, message, request_id, request.url.path)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
return _error_response(400, "Неверный запрос", request_id, request.url.path)
@app.exception_handler(psycopg.OperationalError)
async def db_unavailable_handler(request: Request, exc: psycopg.OperationalError):
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
logger.error("event=db_unavailable type=%s request_id=%s",
type(exc).__name__, request_id)
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
@app.exception_handler(RuntimeError)
async def runtime_error_handler(request: Request, exc: RuntimeError):
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
text = str(exc)
if "db_unavailable" in text:
logger.error("event=db_unavailable request_id=%s", request_id)
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
if "redis_unavailable" in text:
logger.error("event=redis_unavailable request_id=%s", request_id)
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
if "s3_unavailable" in text:
logger.error("event=s3_unavailable request_id=%s", request_id)
return _error_response(503, "Зависимость недоступна", request_id, request.url.path)
logger.error("event=runtime_error type=%s request_id=%s",
type(exc).__name__, request_id)
return _error_response(500, "Внутренняя ошибка", request_id, request.url.path)
@app.exception_handler(Exception)
async def unhandled_exception_handler(request: Request, exc: Exception):
request_id = getattr(request.state, "request_id", uuid.uuid4().hex)
logger.error("event=unhandled_exception type=%s request_id=%s",
type(exc).__name__, request_id)
return _error_response(500, "Внутренняя ошибка", request_id, request.url.path)
app.include_router(health.router)
app.include_router(auth.router)
app.include_router(users.router)
app.include_router(chats.router)
app.include_router(messages.router)
app.include_router(attachments.router)
if __name__ == "__main__":
config = uvicorn.Config(
"app.main:app",
host="0.0.0.0",
port=8080,
log_level="info",
)
server = uvicorn.Server(config)
def _handle_sigterm(signum, frame):
logger.info("event=sigterm_received signal=%s", signum)
server.should_exit = True
signal.signal(signal.SIGTERM, _handle_sigterm)
server.run()
View File
+163
View File
@@ -0,0 +1,163 @@
from datetime import datetime, timezone
from typing import List, Optional
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
def _to_z(dt: datetime) -> str:
"""RFC 3339 UTC с Z, как требует contracts/README.md."""
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
dt = dt.astimezone(timezone.utc)
return dt.isoformat().replace("+00:00", "Z")
def _not_blank(value: str) -> str:
"""Контракт: обязательные строки не состоят только из пробельных символов."""
if value is None or value.strip() == "":
raise ValueError("must not be blank")
return value
class Health(BaseModel):
model_config = ConfigDict(extra="forbid")
status: str
class CreateUser(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("display_name")
@classmethod
def _display_name_not_blank(cls, v: str) -> str:
return _not_blank(v)
@field_validator("password")
@classmethod
def _password_not_blank(cls, v: str) -> str:
return _not_blank(v)
class User(BaseModel):
model_config = ConfigDict(extra="forbid")
id: UUID
username: str
display_name: str
created_at: datetime
@field_serializer("created_at")
def _ser_created_at(self, dt: datetime) -> str:
return _to_z(dt)
class CreateChat(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(min_length=1, max_length=100)
member_ids: List[UUID] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def _title_not_blank(cls, v: str) -> str:
return _not_blank(v)
class Chat(BaseModel):
model_config = ConfigDict(extra="forbid")
id: UUID
title: str
member_ids: List[UUID]
created_at: datetime
@field_serializer("created_at")
def _ser_created_at(self, dt: datetime) -> str:
return _to_z(dt)
class CreateMessage(BaseModel):
"""text может быть пустым, если есть хотя бы одно вложение (lab4)."""
model_config = ConfigDict(extra="forbid")
text: str = Field(default="", max_length=4000)
attachment_ids: List[UUID] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids")
@classmethod
def _unique_attachments(cls, v: List[UUID]) -> List[UUID]:
if len(set(v)) != len(v):
raise ValueError("duplicate attachment_ids")
return v
class Message(BaseModel):
model_config = ConfigDict(extra="forbid")
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: List[UUID] = Field(default_factory=list)
@field_serializer("created_at")
def _ser_created_at(self, dt: datetime) -> str:
return _to_z(dt)
class ChatPage(BaseModel):
model_config = ConfigDict(extra="forbid")
items: List[Chat]
next_cursor: Optional[str] = None
class MessagePage(BaseModel):
model_config = ConfigDict(extra="forbid")
items: List[Message]
next_cursor: Optional[str] = None
class Session(BaseModel):
model_config = ConfigDict(extra="forbid")
token: str
token_type: str = "Bearer"
expires_at: datetime
user: User
@field_serializer("expires_at")
def _ser_expires_at(self, dt: datetime) -> str:
return _to_z(dt)
class Variant(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class Attachment(BaseModel):
model_config = ConfigDict(extra="forbid")
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str # "ready"
variants: List[Variant] = Field(default_factory=list)
error_code: Optional[str] = None
created_at: datetime
updated_at: datetime
@field_serializer("created_at")
def _ser_created_at(self, dt: datetime) -> str:
return _to_z(dt)
@field_serializer("updated_at")
def _ser_updated_at(self, dt: datetime) -> str:
return _to_z(dt)
+128
View File
@@ -0,0 +1,128 @@
"""Outbox worker lab5/lab6: publish неопубликованных job в RabbitMQ с трейсингом.
lab6: payload строки outbox содержит traceparent/tracestate, снятые в момент
upload-запроса API. Восстанавливаем контекст, создаём span job.publish как
child upload-span и инжектим обновлённый traceparent в headers сообщения.
Лог event=outbox_published пишется ВНУТРИ span job.publish, чтобы в Loki
попадал настоящий trace_id (корреляция лог↔trace).
"""
import logging
import signal
import sys
import time
import traceback
from opentelemetry import context as otel_context, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from .storage.db import init_pool, close_pool
from .storage.postgres import storage
from .storage.rabbit import QUEUE, publish_job
from .telemetry import init_telemetry
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
log = logging.getLogger("outbox")
_stop = False
_tracer = trace.get_tracer("outbox")
_propagator = TraceContextTextMapPropagator()
init_telemetry("messenger-outbox", "1.6.0")
def _handle_sigterm(signum, frame):
global _stop
log.info("event=sigterm_received signal=%s", signum)
_stop = True
def _extract_parent(payload: dict) -> otel_context.Context:
carrier: dict[str, str] = {}
if payload.get("traceparent"):
carrier["traceparent"] = str(payload["traceparent"])
if payload.get("tracestate"):
carrier["tracestate"] = str(payload["tracestate"])
if not carrier:
return otel_context.Context()
try:
return _propagator.extract(carrier=carrier)
except Exception as exc: # noqa: BLE001
log.warning("event=trace_parent_parse_failed type=%s", type(exc).__name__)
return otel_context.Context()
def _publish_with_trace(payload: dict, outbox_id: int) -> None:
"""Публикует payload; mark_published и лог — внутри span."""
parent_ctx = _extract_parent(payload)
carrier: dict[str, str] = {}
with _tracer.start_as_current_span(
"job.publish",
context=parent_ctx,
kind=SpanKind.PRODUCER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"job.id": str(payload.get("job_id", "")),
"job.type": str(payload.get("type", "unknown")),
"outbox.id": int(outbox_id),
},
) as span:
try:
try:
_propagator.inject(carrier)
except Exception as exc: # noqa: BLE001
log.warning("event=trace_inject_failed type=%s", type(exc).__name__)
carrier = {}
publish_job(payload, extra_headers=carrier)
storage.mark_outbox_published(outbox_id)
# Лог внутри span → record.trace_id валидный.
log.info("event=outbox_published id=%s job_id=%s",
outbox_id, payload.get("job_id"))
span.set_status(Status(StatusCode.OK))
except Exception as exc:
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
raise
def main() -> None:
signal.signal(signal.SIGTERM, _handle_sigterm)
init_pool()
log.info("event=outbox_start")
while not _stop:
try:
rows = storage.list_unpublished_outbox(limit=50)
except Exception as exc: # noqa: BLE001
log.error("event=outbox_select_failed type=%s msg=%s",
type(exc).__name__, str(exc)[:300])
time.sleep(2)
continue
for outbox_id, payload in rows:
if _stop:
break
try:
_publish_with_trace(payload, outbox_id)
except Exception as exc: # noqa: BLE001
log.warning(
"event=outbox_publish_failed id=%s type=%s msg=%s",
outbox_id, type(exc).__name__, str(exc)[:300],
)
time.sleep(2)
break
time.sleep(2)
close_pool()
log.info("event=outbox_stop")
if __name__ == "__main__":
try:
main()
except Exception:
log.error("event=outbox_fatal\n%s", traceback.format_exc())
sys.exit(1)
View File
+212
View File
@@ -0,0 +1,212 @@
import os
from contextlib import contextmanager
from typing import Iterator
import psycopg
from psycopg_pool import ConnectionPool
_pool: ConnectionPool | None = None
def _dsn() -> str:
dsn = os.environ.get("DATABASE_URL")
if not dsn:
raise RuntimeError("DATABASE_URL is not set")
return dsn
def init_pool() -> None:
"""Ограниченный пул соединений с таймаутами. Вызывается один раз на старте."""
global _pool
if _pool is not None:
return
_pool = ConnectionPool(
conninfo=_dsn(),
min_size=2,
max_size=10,
# Короткий таймаут получения соединения из пула: readiness/бизнес-запрос
# при мёртвой БД должен быстро вернуть 503, а не висеть.
timeout=3.0,
max_idle=300.0,
# check: перед выдачей соединения проверяем его живость; мёртвые
# соединения после рестарта PostgreSQL отбрасываются, пул сам
# открывает новые — readiness восстанавливается без рестарта API.
check=ConnectionPool.check_connection,
# Раз в 30с фоновый worker пула добивает min_size живыми соединениями,
# чтобы после старта БД готовность вернулась в пределах окна теста.
reconnect_timeout=30.0,
kwargs={"options": "-c statement_timeout=5000"},
open=True,
)
def close_pool() -> None:
global _pool
if _pool is not None:
_pool.close()
_pool = None
@contextmanager
def connection() -> Iterator[psycopg.Connection]:
"""Соединение из пула; autocommit=False для явных транзакций."""
if _pool is None:
init_pool()
assert _pool is not None
with _pool.connection() as conn:
yield conn
SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_messages_chat_created
ON messages (chat_id, created_at, id);
-- sessions остаются как fallback, если SESSION_STORE=postgres.
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at TIMESTAMPTZ NOT NULL
);
-- lab3: общая идемпотентность POST /messages.
-- scope: user_id + method + path + key (см. contracts/README.md).
-- body_hash — sha256 канонизированного JSON без attachment_ids=[], чтобы
-- отличать повтор «того же» body (201, тот же Message) от «другого» (409).
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method TEXT NOT NULL,
path TEXT NOT NULL,
key TEXT NOT NULL,
body_hash TEXT NOT NULL,
response_status INT NOT NULL,
response_body JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_created
ON idempotency_keys (created_at);
-- lab4: вложения
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 TEXT NOT NULL,
object_key TEXT NOT NULL UNIQUE,
status TEXT NOT NULL, -- queued/processing/ready/failed
error_code TEXT,
processing_until TIMESTAMPTZ, -- lease для worker (lab5)
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat
ON attachments (chat_id);
-- связь сообщение↔вложение (порядок сохраняем)
CREATE TABLE IF NOT EXISTS message_attachments (
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
position INT NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX IF NOT EXISTS idx_msg_att_message
ON message_attachments (message_id, position);
-- lab5: варианты обработки (идемпотентность по (attachment_id, pipeline_version, variant))
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INT NOT NULL,
variant TEXT NOT NULL,
object_key TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
width INT NOT NULL,
height INT NOT NULL,
sha256 TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
-- lab5 level 4: transactional outbox (разрыв COMMIT → publish)
CREATE TABLE IF NOT EXISTS outbox (
id BIGSERIAL PRIMARY KEY,
job_id UUID NOT NULL UNIQUE,
payload JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished
ON outbox (id) WHERE published_at IS NULL;
"""
def init_schema() -> None:
"""Идемпотентная миграция: безопасно вызывать многократно.
Вызывается ОДИН РАЗ отдельным one-shot сервисом migrate
(см. compose.yaml). Реплики API миграции больше не выполняют:
они ждут service_completed_successfully.
"""
with connection() as conn:
with conn.cursor() as cur:
cur.execute(SCHEMA_SQL)
conn.commit()
def run_migrations() -> None:
"""Точка входа для one-shot migrate-контейнера.
Поднимает пул, применяет схему, закрывает пул. Завершается с кодом 0
при успехе — этого достаточно для service_completed_successfully.
"""
init_pool()
try:
init_schema()
finally:
close_pool()
if __name__ == "__main__":
# Запуск: python -m app.storage.db
# (в compose.yaml — command: ["python", "-m", "app.storage.db"])
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
log = logging.getLogger("migrate")
log.info("event=migrations_start")
run_migrations()
log.info("event=migrations_done")
+177
View File
@@ -0,0 +1,177 @@
import hashlib
import hmac
import secrets
import threading
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional
from uuid import UUID, uuid4
from ..models.schemas import Chat, Message, User
def now_utc() -> datetime:
"""Время сервера в UTC с суффиксом Z на выходе."""
return datetime.now(timezone.utc)
# --- Password KDF (stdlib scrypt) ---
_SCRYPT_N = 2 ** 14
_SCRYPT_R = 8
_SCRYPT_P = 1
_SCRYPT_DKLEN = 32
def _hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
dk = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=_SCRYPT_N,
r=_SCRYPT_R,
p=_SCRYPT_P,
dklen=_SCRYPT_DKLEN,
)
return "scrypt$%d$%d$%d$%s$%s" % (
_SCRYPT_N, _SCRYPT_R, _SCRYPT_P,
salt.hex(), dk.hex(),
)
def _verify_password(password: str, stored: str) -> bool:
try:
scheme, n, r, p, salt_hex, dk_hex = stored.split("$")
if scheme != "scrypt":
return False
dk = hashlib.scrypt(
password.encode("utf-8"),
salt=bytes.fromhex(salt_hex),
n=int(n), r=int(r), p=int(p),
dklen=len(bytes.fromhex(dk_hex)),
)
return hmac.compare_digest(dk.hex(), dk_hex)
except Exception:
return False
class MemoryStorage:
"""Потокобезопасное In-memory хранилище пользователей, чатов, сообщений и сессий. Все данные исчезают при перезапуске процесса"""
def __init__(self) -> None:
self._lock = threading.RLock()
self._users: Dict[UUID, User] = {}
self._usernames: Dict[str, UUID] = {}
self._passwords: Dict[UUID, str] = {}
self._chats: Dict[UUID, Chat] = {}
self._messages: Dict[UUID, List[Message]] = {}
# token -> (user_id, expires_at)
self._sessions: Dict[str, tuple] = {}
# ---------- Пользователи ----------
def create_user(self, username: str, display_name: str, password: str) -> User:
with self._lock:
if username in self._usernames:
raise ValueError("username_taken")
user = User(
id=uuid4(),
username=username,
display_name=display_name,
created_at=now_utc(),
)
self._users[user.id] = user
self._usernames[username] = user.id
self._passwords[user.id] = _hash_password(password)
return user
def get_user(self, user_id: UUID) -> Optional[User]:
with self._lock:
return self._users.get(user_id)
def get_user_by_username(self, username: str) -> Optional[User]:
with self._lock:
uid = self._usernames.get(username)
return self._users.get(uid) if uid else None
def verify_credentials(self, username: str, password: str) -> Optional[User]:
with self._lock:
uid = self._usernames.get(username)
if uid is None:
return None
stored = self._passwords.get(uid)
if not stored or not _verify_password(password, stored):
return None
return self._users.get(uid)
# ---------- Сессии ----------
def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple:
with self._lock:
token = secrets.token_urlsafe(32)
expires_at = now_utc() + timedelta(seconds=ttl_seconds)
self._sessions[token] = (user_id, expires_at)
return token, expires_at
def get_session_user(self, token: str) -> Optional[User]:
with self._lock:
entry = self._sessions.get(token)
if entry is None:
return None
user_id, expires_at = entry
if expires_at <= now_utc():
self._sessions.pop(token, None)
return None
return self._users.get(user_id)
def revoke_session(self, token: str) -> None:
with self._lock:
self._sessions.pop(token, None)
# ---------- Чаты ----------
def create_chat(self, title: str, member_ids: List[UUID]) -> Chat:
with self._lock:
for uid in member_ids:
if uid not in self._users:
raise ValueError("unknown_member")
chat = Chat(
id=uuid4(),
title=title,
member_ids=list(member_ids),
created_at=now_utc(),
)
self._chats[chat.id] = chat
self._messages[chat.id] = []
return chat
def get_chat(self, chat_id: UUID) -> Optional[Chat]:
with self._lock:
return self._chats.get(chat_id)
def list_chats_for_user(self, user_id: UUID) -> List[Chat]:
with self._lock:
chats = [c for c in self._chats.values() if user_id in c.member_ids]
chats.sort(key=lambda c: (c.created_at, str(c.id)))
return chats
# ---------- Сообщения ----------
def add_message(self, chat_id: UUID, sender_id: UUID, text: str) -> Message:
"""Улучшение Трека 1: список сообщений хранится уже отсортированным.
created_at монотонен в пределах одного процесса, поэтому append
сохраняет порядок (created_at, id) по возрастанию — сортировка на
чтении не нужна. При конкурентных вставках под RLock порядок
сохраняется, но монотонность часов не гарантируется контрактом."""
with self._lock:
message = Message(
id=uuid4(),
chat_id=chat_id,
sender_id=sender_id,
text=text,
created_at=now_utc(),
)
self._messages.setdefault(chat_id, []).append(message)
return message
def list_messages(self, chat_id: UUID) -> List[Message]:
with self._lock:
# Без пересортировки: список уже в порядке (created_at, id).
return list(self._messages.get(chat_id, []))
storage = MemoryStorage() # единственный экземпляр хранилища на процесс
+614
View File
@@ -0,0 +1,614 @@
import hashlib
import hmac
import json as _json
import secrets
import threading
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from uuid import UUID, uuid4
from ..models.schemas import Attachment, Chat, Message, User, Variant
from .db import connection
from .sessions import (
create_session as _sess_create,
get_session_user as _sess_get,
revoke_session as _sess_revoke,
)
def now_utc() -> datetime:
return datetime.now(timezone.utc)
# --- Password KDF (stdlib scrypt) ---
_SCRYPT_N = 2 ** 14
_SCRYPT_R = 8
_SCRYPT_P = 1
_SCRYPT_DKLEN = 32
def _hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
dk = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=_SCRYPT_N, r=_SCRYPT_R, p=_SCRYPT_P, dklen=_SCRYPT_DKLEN,
)
return "scrypt$%d$%d$%d$%s$%s" % (
_SCRYPT_N, _SCRYPT_R, _SCRYPT_P, salt.hex(), dk.hex(),
)
def _verify_password(password: str, stored: str) -> bool:
try:
scheme, n, r, p, salt_hex, dk_hex = stored.split("$")
if scheme != "scrypt":
return False
dk = hashlib.scrypt(
password.encode("utf-8"),
salt=bytes.fromhex(salt_hex),
n=int(n), r=int(r), p=int(p),
dklen=len(bytes.fromhex(dk_hex)),
)
return hmac.compare_digest(dk.hex(), dk_hex)
except Exception:
return False
def _row_to_user(row) -> User:
return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3])
def _row_to_chat(row, member_ids: List[UUID]) -> Chat:
return Chat(id=row[0], title=row[1], member_ids=member_ids, created_at=row[2])
def _row_to_message(row, attachment_ids: List[UUID]) -> Message:
return Message(
id=row[0], chat_id=row[1], sender_id=row[2],
text=row[3], created_at=row[4],
attachment_ids=attachment_ids,
)
def _row_to_attachment(row, variants: Optional[list] = None) -> Attachment:
# row: id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
# status, error_code, created_at, updated_at
return Attachment(
id=row[0], chat_id=row[1], owner_id=row[2],
filename=row[3], content_type=row[4], size_bytes=row[5],
sha256=row[6], status=row[7], variants=variants or [],
error_code=row[8],
created_at=row[9], updated_at=row[10],
)
class PostgresStorage:
"""Потокобезопасная обёртка; параллелизм обеспечивает PostgreSQL."""
def __init__(self) -> None:
self._lock = threading.RLock()
# ---------- Пользователи ----------
def create_user(self, username: str, display_name: str, password: str) -> User:
with self._lock, connection() as conn:
with conn.cursor() as cur:
cur.execute("SELECT 1 FROM users WHERE username = %s", (username,))
if cur.fetchone() is not None:
raise ValueError("username_taken")
user = User(
id=uuid4(), username=username,
display_name=display_name, created_at=now_utc(),
)
try:
cur.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (%s, %s, %s, %s, %s)",
(user.id, user.username, user.display_name,
_hash_password(password), user.created_at),
)
except Exception as exc:
if exc.__class__.__name__ == "UniqueViolation":
raise ValueError("username_taken")
raise
conn.commit()
return user
def get_user(self, user_id: UUID) -> Optional[User]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
)
row = cur.fetchone()
return _row_to_user(row) if row else None
def get_user_by_username(self, username: str) -> Optional[User]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, username, display_name, created_at FROM users WHERE username = %s",
(username,),
)
row = cur.fetchone()
return _row_to_user(row) if row else None
def verify_credentials(self, username: str, password: str) -> Optional[User]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, username, display_name, created_at, password_hash"
" FROM users WHERE username = %s",
(username,),
)
row = cur.fetchone()
if row is None:
return None
if not _verify_password(password, row[4]):
return None
return _row_to_user(row[:4])
# ---------- Сессии ----------
def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple:
return _sess_create(user_id, ttl_seconds)
def get_session_user(self, token: str) -> Optional[User]:
return _sess_get(token)
def revoke_session(self, token: str) -> None:
_sess_revoke(token)
# ---------- Чаты ----------
def create_chat(self, title: str, member_ids: List[UUID]) -> Chat:
with self._lock, connection() as conn:
with conn.cursor() as cur:
# проверяем всех участников
cur.execute(
"SELECT id FROM users WHERE id = ANY(%s)",
(list(member_ids),),
)
found = {r[0] for r in cur.fetchall()}
if len(found) != len(set(member_ids)):
raise ValueError("unknown_member")
chat = Chat(
id=uuid4(), title=title,
member_ids=list(member_ids), created_at=now_utc(),
)
cur.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(chat.id, chat.title, chat.created_at),
)
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat.id, uid) for uid in member_ids],
)
conn.commit()
return chat
def get_chat(self, chat_id: UUID) -> Optional[Chat]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, title, created_at FROM chats WHERE id = %s",
(chat_id,),
)
row = cur.fetchone()
if row is None:
return None
cur.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id,),
)
members = [r[0] for r in cur.fetchall()]
return _row_to_chat(row, members)
def list_chats_for_user(self, user_id: UUID) -> List[Chat]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT c.id, c.title, c.created_at FROM chats c"
" JOIN chat_members m ON m.chat_id = c.id"
" WHERE m.user_id = %s ORDER BY c.created_at, c.id",
(user_id,),
)
rows = cur.fetchall()
chats = []
for row in rows:
cur.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s", (row[0],),
)
members = [r[0] for r in cur.fetchall()]
chats.append(_row_to_chat(row, members))
return chats
# ---------- Сообщения ----------
def _load_message_attachments(self, cur, message_ids: List[UUID]) -> dict[UUID, List[UUID]]:
if not message_ids:
return {}
cur.execute(
"SELECT message_id, attachment_id, position FROM message_attachments"
" WHERE message_id = ANY(%s) ORDER BY message_id, position",
(list(message_ids),),
)
result: dict[UUID, List[UUID]] = {}
for mid, aid, _pos in cur.fetchall():
result.setdefault(mid, []).append(aid)
return result
def list_messages_page(
self, chat_id: UUID, limit: int, after: Optional[tuple] = None,
) -> List[Message]:
with connection() as conn, conn.cursor() as cur:
if after is None:
cur.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages"
" WHERE chat_id = %s ORDER BY created_at, id LIMIT %s",
(chat_id, limit),
)
else:
after_ts, after_id = after
cur.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages"
" WHERE chat_id = %s AND (created_at, id) > (%s, %s)"
" ORDER BY created_at, id LIMIT %s",
(chat_id, after_ts, after_id, limit),
)
rows = cur.fetchall()
ids = [r[0] for r in rows]
att_map = self._load_message_attachments(cur, ids)
return [_row_to_message(r, att_map.get(r[0], [])) for r in rows]
# ---------- Идемпотентность (lab3) ----------
def add_message_idempotent(
self,
chat_id: UUID,
sender_id: UUID,
text: str,
attachment_ids: List[UUID],
user_id: UUID,
method: str,
path: str,
key: str,
body_hash: str,
) -> tuple[str, Optional[Message], Optional[dict]]:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO idempotency_keys"
" (user_id, method, path, key, body_hash, response_status, response_body)"
" VALUES (%s, %s, %s, %s, %s, 0, '{}'::jsonb)"
" ON CONFLICT (user_id, method, path, key) DO NOTHING"
" RETURNING key",
(user_id, method, path, key, body_hash),
)
won = cur.fetchone() is not None
if won:
message = Message(
id=uuid4(), chat_id=chat_id, sender_id=sender_id,
text=text, created_at=now_utc(),
attachment_ids=list(attachment_ids),
)
cur.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" VALUES (%s, %s, %s, %s, %s)",
(message.id, message.chat_id, message.sender_id,
message.text, message.created_at),
)
if attachment_ids:
cur.executemany(
"INSERT INTO message_attachments"
" (message_id, attachment_id, position)"
" VALUES (%s, %s, %s)",
[(message.id, aid, idx)
for idx, aid in enumerate(attachment_ids)],
)
body_json = _json.loads(message.model_dump_json())
cur.execute(
"UPDATE idempotency_keys"
" SET response_status = %s, response_body = %s"
" WHERE user_id = %s AND method = %s AND path = %s AND key = %s",
(201, _json.dumps(body_json),
user_id, method, path, key),
)
conn.commit()
return ("created", message, None)
cur.execute(
"SELECT body_hash, response_status, response_body"
" FROM idempotency_keys"
" WHERE user_id = %s AND method = %s AND path = %s AND key = %s",
(user_id, method, path, key),
)
row = cur.fetchone()
conn.commit()
if row is None:
return ("retry", None, None)
return (
"replay",
None,
{
"body_hash": row[0],
"response_status": row[1],
"response_body": row[2],
},
)
# ---------- Вложения ----------
def create_attachment(self, attachment: Attachment, object_key: str) -> Attachment:
"""lab4 fallback (синхронный ready). Не используется в lab5."""
with connection() as conn, conn.cursor() as cur:
cur.execute(
"INSERT INTO attachments"
" (id, chat_id, owner_id, filename, content_type, size_bytes,"
" sha256, object_key, status, error_code, created_at, updated_at)"
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
attachment.id, attachment.chat_id, attachment.owner_id,
attachment.filename, attachment.content_type,
attachment.size_bytes, attachment.sha256, object_key,
attachment.status, attachment.error_code,
attachment.created_at, attachment.updated_at,
),
)
conn.commit()
return attachment
def create_attachment_queued(
self,
attachment: Attachment,
staging_key: str,
outbox_payload: dict,
) -> Attachment:
"""lab5: attachment в queued + строка outbox в ОДНОЙ транзакции."""
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO attachments"
" (id, chat_id, owner_id, filename, content_type, size_bytes,"
" sha256, object_key, status, error_code, processing_until,"
" created_at, updated_at)"
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
attachment.id, attachment.chat_id, attachment.owner_id,
attachment.filename, attachment.content_type,
attachment.size_bytes, attachment.sha256, staging_key,
"queued", None, None,
attachment.created_at, attachment.updated_at,
),
)
cur.execute(
"INSERT INTO outbox (job_id, payload) VALUES (%s, %s)",
(outbox_payload["job_id"], _json.dumps(outbox_payload)),
)
conn.commit()
return attachment
def claim_processing(self, attachment_id: UUID, lease_seconds: int = 60) -> bool:
with connection() as conn:
with conn.cursor() as cur:
now = now_utc()
cur.execute(
"UPDATE attachments SET status='processing',"
" processing_until=%s, updated_at=%s"
" WHERE id=%s AND status IN ('queued','processing')"
" RETURNING id",
(now + timedelta(seconds=lease_seconds), now, attachment_id),
)
won = cur.fetchone() is not None
conn.commit()
return won
def recover_stuck_processing(self) -> int:
with connection() as conn:
with conn.cursor() as cur:
now = now_utc()
cur.execute(
"UPDATE attachments SET status='queued', processing_until=NULL,"
" updated_at=%s"
" WHERE status='processing' AND processing_until IS NOT NULL"
" AND processing_until < %s",
(now, now),
)
count = cur.rowcount
conn.commit()
return count
def replay_failed_to_queued(self, attachment_id: UUID) -> str:
"""Операторский replay: failed → queued с новой записью в outbox.
Возвращает:
- "requeued" — статус был failed, переведён в queued и запланирован job;
- "already" — запись уже queued/processing/ready (защита готовых), ничего не меняем;
- "missing" — такого вложения нет.
Ключ эффекта (attachment_id, pipeline_version, variant) и ON CONFLICT
в finalize_ready гарантируют, что повтор не создаст дублей вариантов.
Новый job_id и строка outbox публикуются отдельным процессом outbox,
поэтому разрыв между COMMIT и publish закрыт тем же механизмом, что и при upload.
"""
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT status FROM attachments WHERE id = %s FOR UPDATE",
(attachment_id,),
)
row = cur.fetchone()
if row is None:
conn.rollback()
return "missing"
status = row[0]
if status != "failed":
conn.rollback()
return "already"
now = now_utc()
cur.execute(
"UPDATE attachments SET status='queued', error_code=NULL,"
" processing_until=NULL, updated_at=%s WHERE id=%s",
(now, attachment_id),
)
job_id = uuid4()
payload = {
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"created_at": now.isoformat().replace("+00:00", "Z"),
}
cur.execute(
"INSERT INTO outbox (job_id, payload) VALUES (%s, %s)",
(job_id, _json.dumps(payload)),
)
conn.commit()
return "requeued"
def finalize_ready(
self,
attachment_id: UUID,
pipeline_version: int,
variants: list[dict],
) -> None:
with connection() as conn:
with conn.cursor() as cur:
original = next(v for v in variants if v["name"] == "original")
cur.execute(
"UPDATE attachments SET status='ready', error_code=NULL,"
" object_key=%s, processing_until=NULL, updated_at=%s"
" WHERE id=%s",
(original["object_key"], now_utc(), attachment_id),
)
for v in variants:
if v["name"] == "original":
continue
cur.execute(
"INSERT INTO attachment_variants"
" (attachment_id, pipeline_version, variant, object_key,"
" content_type, size_bytes, width, height, sha256)"
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
" ON CONFLICT (attachment_id, pipeline_version, variant)"
" DO NOTHING",
(
attachment_id, pipeline_version, v["name"],
v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"],
),
)
conn.commit()
def finalize_failed(self, attachment_id: UUID, error_code: str) -> None:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE attachments SET status='failed', error_code=%s,"
" processing_until=NULL, updated_at=%s WHERE id=%s",
(error_code, now_utc(), attachment_id),
)
conn.commit()
def get_attachment(self, attachment_id: UUID) -> Optional[Attachment]:
pair = self.get_attachment_with_key(attachment_id)
return pair[0] if pair else None
def get_attachment_with_key(
self, attachment_id: UUID
) -> Optional[tuple[Attachment, str]]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes,"
" sha256, status, error_code, created_at, updated_at, object_key"
" FROM attachments WHERE id = %s",
(attachment_id,),
)
row = cur.fetchone()
if row is None:
return None
att = _row_to_attachment(row[:11], self.get_attachment_variants(attachment_id))
return att, row[11]
def list_attachments_by_ids(
self, ids: List[UUID], chat_id: UUID
) -> dict[UUID, tuple[Attachment, str]]:
if not ids:
return {}
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes,"
" sha256, status, error_code, created_at, updated_at, object_key"
" FROM attachments WHERE chat_id = %s AND id = ANY(%s)",
(chat_id, list(ids)),
)
result: dict[UUID, tuple[Attachment, str]] = {}
for row in cur.fetchall():
att = _row_to_attachment(row[:11])
result[row[0]] = (att, row[11])
return result
def delete_attachment(self, attachment_id: UUID) -> None:
with connection() as conn, conn.cursor() as cur:
cur.execute("DELETE FROM attachments WHERE id = %s", (attachment_id,))
conn.commit()
def get_attachment_variants(self, attachment_id: UUID) -> list[dict]:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT variant, content_type, size_bytes, width, height, sha256"
" FROM attachment_variants WHERE attachment_id=%s"
" ORDER BY variant",
(attachment_id,),
)
return [
{
"name": r[0], "content_type": r[1], "size_bytes": r[2],
"width": r[3], "height": r[4], "sha256": r[5],
}
for r in cur.fetchall()
]
def get_staging_key(self, attachment_id: UUID) -> Optional[str]:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT object_key FROM attachments WHERE id=%s",
(attachment_id,),
)
row = cur.fetchone()
return row[0] if row else None
# ---------- Outbox (lab5 level 4) ----------
def list_unpublished_outbox(self, limit: int = 100) -> list[tuple[int, dict]]:
"""Возвращает [(outbox_id, payload_dict), ...].
Всегда dict; если payload в БД пришёл строкой (например, из старой
версии), распаковываем через json.loads.
"""
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT id, payload FROM outbox WHERE published_at IS NULL"
" ORDER BY id LIMIT %s",
(limit,),
)
rows = cur.fetchall()
result: list[tuple[int, dict]] = []
for r in rows:
raw = r[1]
if isinstance(raw, dict):
payload = raw
else:
payload = _json.loads(raw)
if not isinstance(payload, dict):
# защита от мусора — пропустить
continue
result.append((int(r[0]), payload))
return result
def mark_outbox_published(self, outbox_id: int) -> None:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE outbox SET published_at=now() WHERE id=%s",
(outbox_id,),
)
conn.commit()
storage = PostgresStorage() # единственный экземпляр на процесс
+155
View File
@@ -0,0 +1,155 @@
"""RabbitMQ: topology, publisher with confirms, consumer с manual ack.
- Основная очередь: durable quorum queue.
- Retry: TTL-очереди с dead-letter обратно в основную; x-retry-count.
- DLQ: финальные ошибки и исчерпанные retry.
- Publisher confirms: publish() ждёт подтверждения брокера.
- lab6: extra_headers для traceparent/tracestate.
"""
import json
import logging
import os
import threading
from typing import Optional
import pika
log = logging.getLogger("rabbit")
EXCHANGE = "messenger.attachments"
QUEUE = "attachment.process.v1"
RETRY_EXCHANGE = "messenger.attachments.retry"
DLX = "messenger.attachments.dlx"
DLQ = "attachment.process.v1.dlq"
RETRY_DELAYS = [5, 15, 60]
MAX_ATTEMPTS = len(RETRY_DELAYS) + 1
_topology_ready = False
_topology_lock = threading.Lock()
def _url() -> str:
return os.environ.get("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/%2F")
def connection_params() -> pika.ConnectionParameters:
return pika.URLParameters(_url())
def connect() -> pika.BlockingConnection:
return pika.BlockingConnection(connection_params())
def setup_topology(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
channel.exchange_declare(RETRY_EXCHANGE, exchange_type="direct", durable=True)
channel.exchange_declare(DLX, exchange_type="direct", durable=True)
channel.queue_declare(DLQ, durable=True, arguments={"x-queue-type": "quorum"})
channel.queue_bind(DLQ, DLX, routing_key=DLQ)
channel.queue_declare(
QUEUE,
durable=True,
arguments={
"x-queue-type": "quorum",
"x-dead-letter-exchange": DLX,
"x-dead-letter-routing-key": DLQ,
},
)
channel.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE)
for delay in RETRY_DELAYS:
qname = f"attachment.retry.{delay}s"
channel.queue_declare(
qname,
durable=True,
arguments={
"x-message-ttl": delay * 1000,
"x-dead-letter-exchange": EXCHANGE,
"x-dead-letter-routing-key": QUEUE,
},
)
channel.queue_bind(qname, RETRY_EXCHANGE, routing_key=qname)
def _ensure_topology(ch: pika.adapters.blocking_connection.BlockingChannel) -> None:
global _topology_ready
if _topology_ready:
return
with _topology_lock:
if _topology_ready:
return
setup_topology(ch)
_topology_ready = True
def publish_job(payload: dict, extra_headers: Optional[dict] = None) -> None:
"""Публикация job с publisher confirm. extra_headers — traceparent."""
conn = connect()
try:
ch = conn.channel()
ch.confirm_delivery()
_ensure_topology(ch)
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
headers = dict(extra_headers or {})
ch.basic_publish(
exchange=EXCHANGE,
routing_key=QUEUE,
body=body.encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
headers=headers or None,
),
mandatory=True,
)
finally:
conn.close()
def publish_retry(payload: dict, retry_count: int) -> None:
idx = min(retry_count - 1, len(RETRY_DELAYS) - 1)
delay = RETRY_DELAYS[idx]
qname = f"attachment.retry.{delay}s"
conn = connect()
try:
ch = conn.channel()
ch.confirm_delivery()
_ensure_topology(ch)
headers = dict(payload.get("_headers") or {})
headers["x-retry-count"] = retry_count
clean = {k: v for k, v in payload.items() if k != "_headers"}
body = json.dumps(clean, ensure_ascii=False, separators=(",", ":"))
ch.basic_publish(
exchange=RETRY_EXCHANGE,
routing_key=qname,
body=body.encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
headers=headers,
),
mandatory=True,
)
finally:
conn.close()
def publish_dlq(payload: dict) -> None:
conn = connect()
try:
ch = conn.channel()
ch.confirm_delivery()
_ensure_topology(ch)
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
ch.basic_publish(
exchange=DLX,
routing_key=DLQ,
body=body.encode("utf-8"),
properties=pika.BasicProperties(delivery_mode=2, content_type="application/json"),
mandatory=True,
)
finally:
conn.close()
+88
View File
@@ -0,0 +1,88 @@
"""Операторский replay failed-задач: failed → queued + запись в outbox.
Использование:
python -m app.storage.replay_failed <attachment_id> [<attachment_id> ...]
python -m app.storage.replay_failed --all-failed
Защита:
- переводит ТОЛЬКО failed; queued/processing/ready не трогает (возвращает "already");
- новая строка outbox с новым job_id — публикация через существующий outbox-процесс;
- идемпотентность эффекта по (attachment_id, pipeline_version, variant)
не даёт создать дубли вариантов, если повторная обработка уже была выполнена.
Логи пишутся только с ID и статусом, без секретов и содержимого файлов.
"""
import logging
import sys
import uuid
from .db import init_pool, close_pool
from .postgres import storage
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
log = logging.getLogger("replay_failed")
def _parse_args(argv):
if not argv:
print(
"usage: python -m app.storage.replay_failed <attachment_id> [...] | --all-failed",
file=sys.stderr,
)
return None
if argv == ["--all-failed"]:
return "all"
ids = []
for raw in argv:
try:
ids.append(uuid.UUID(raw))
except ValueError:
print("invalid attachment_id: %r" % raw, file=sys.stderr)
return None
return ids
def _list_failed_ids() -> list:
from .db import connection
with connection() as conn, conn.cursor() as cur:
cur.execute("SELECT id FROM attachments WHERE status='failed' ORDER BY updated_at")
return [r[0] for r in cur.fetchall()]
def main(argv) -> int:
parsed = _parse_args(argv)
if parsed is None:
return 2
init_pool()
try:
if parsed == "all":
ids = _list_failed_ids()
else:
ids = parsed
requeued = already = missing = 0
for aid in ids:
result = storage.replay_failed_to_queued(aid)
if result == "requeued":
requeued += 1
log.info("event=replay_requeued attachment_id=%s", aid)
elif result == "already":
already += 1
log.info("event=replay_skipped_not_failed attachment_id=%s", aid)
else:
missing += 1
log.info("event=replay_missing attachment_id=%s", aid)
log.info(
"event=replay_done requested=%d requeued=%d skipped=%d missing=%d",
len(ids), requeued, already, missing,
)
return 0 if missing == 0 else 1
finally:
close_pool()
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+98
View File
@@ -0,0 +1,98 @@
"""S3-клиент (MinIO) для lab4/lab5.
Object key генерирует сервер:
staging: staging/<chat_id>/<attachment_id>/original
final: attachments/<chat_id>/<attachment_id>/original
variant: attachments/<chat_id>/<attachment_id>/<variant>
filename используется только для отображения, не в ключе.
"""
import os
from typing import Optional
from uuid import UUID
import boto3
from botocore.client import Config
from botocore.exceptions import ClientError
def _endpoint() -> str:
return os.environ["S3_ENDPOINT"]
def _bucket() -> str:
return os.environ.get("S3_BUCKET", "messenger-attachments")
def client():
"""Клиент с прикладными credentials (не admin).
Явные короткие таймауты: connect/read — чтобы при недоступности S3
запросы падали быстро и LB отдавал 503, а не 504 по своему timeout.
"""
return boto3.client(
"s3",
endpoint_url=_endpoint(),
region_name=os.environ.get("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ["S3_APP_KEY_ID"],
aws_secret_access_key=os.environ["S3_APP_SECRET_KEY"],
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=2,
read_timeout=3,
retries={"max_attempts": 0},
),
)
def staging_key(chat_id: UUID, attachment_id: UUID) -> str:
"""Staging — приватный prefix, worker читает отсюда."""
return "staging/%s/%s/original" % (chat_id, attachment_id)
def original_key(chat_id: UUID, attachment_id: UUID) -> str:
"""Финальный ключ оригинала. Не зависит от filename."""
return "attachments/%s/%s/original" % (chat_id, attachment_id)
def variant_key(chat_id: UUID, attachment_id: UUID, variant: str) -> str:
return "attachments/%s/%s/%s" % (chat_id, attachment_id, variant)
def put_bytes(key: str, data: bytes, content_type: str) -> None:
client().put_object(
Bucket=_bucket(),
Key=key,
Body=data,
ContentType=content_type,
ContentLength=len(data),
)
def get_object(key: str):
"""Возвращает (bytes, content_type). Бросает ClientError при отсутствии."""
resp = client().get_object(Bucket=_bucket(), Key=key)
body = resp["Body"].read()
ctype = resp.get("ContentType") or "application/octet-stream"
return body, ctype
def delete_object(key: str) -> None:
client().delete_object(Bucket=_bucket(), Key=key)
def head_object(key: str) -> Optional[dict]:
try:
return client().head_object(Bucket=_bucket(), Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
return None
raise
def list_objects(prefix: str = "attachments/"):
"""Итератор по объектам bucket. Используется cleanup-сервисом."""
paginator = client().get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=_bucket(), Prefix=prefix):
for obj in page.get("Contents", []):
yield obj
+176
View File
@@ -0,0 +1,176 @@
"""Фоновая сверка S3 и PostgreSQL (уровень 4 lab4/lab5).
- Orphan: объект в S3 есть, а метаданных/вариантов в БД нет.
Удаляем только если объект старше ORPHAN_GRACE_SECONDS.
- Missing: метаданные есть, объекта в S3 нет — warning.
- Staging: объекты staging/ старше grace удаляются (не привязаны к текущей работе).
- Также периодически возвращает зависшие processing в queued.
"""
import logging
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from . import s3
from .db import connection, init_pool, close_pool
from .postgres import storage
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
log = logging.getLogger("s3_cleanup")
def _interval() -> int:
try:
return max(30, int(os.environ.get("CLEANUP_INTERVAL_SECONDS", "300")))
except ValueError:
return 300
def _grace() -> int:
try:
return max(0, int(os.environ.get("ORPHAN_GRACE_SECONDS", "3600")))
except ValueError:
return 3600
def _known_keys() -> set[str]:
"""Все ожидаемые object key: оригиналы + варианты."""
keys: set[str] = set()
with connection() as conn, conn.cursor() as cur:
cur.execute("SELECT id, chat_id, object_key FROM attachments")
for aid, cid, okey in cur.fetchall():
keys.add(okey)
keys.add(s3.original_key(cid, aid))
cur.execute("SELECT attachment_id, variant FROM attachment_variants")
# нужен chat_id для variant_key
# variants: отдельным запросом с join
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT a.chat_id, v.attachment_id, v.variant"
" FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id"
)
for cid, aid, variant in cur.fetchall():
keys.add(s3.variant_key(cid, aid, variant))
return keys
def _active_attachment_ids() -> set[str]:
"""ID вложений в статусах queued/processing — их staging трогать нельзя."""
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id FROM attachments WHERE status IN ('queued', 'processing')"
)
return {str(r[0]) for r in cur.fetchall()}
def _run_once() -> dict:
now = datetime.now(timezone.utc)
grace_cutoff = now - timedelta(seconds=_grace())
known = _known_keys()
active_ids = _active_attachment_ids()
orphan_deleted = 0
missing = 0
scanned = 0
staging_deleted = 0
# attachments/ — orphan по known
for obj in s3.list_objects("attachments/"):
scanned += 1
key = obj["Key"]
if key in known:
continue
# Orphan: в S3 есть, в БД нет. Удаляем только старше grace period.
lm = obj.get("LastModified")
if lm is None:
continue
if lm.tzinfo is None:
lm = lm.replace(tzinfo=timezone.utc)
if lm > grace_cutoff:
# Свежий объект — возможно in-flight upload; не трогаем.
continue
try:
s3.delete_object(key)
orphan_deleted += 1
log.info("event=orphan_deleted key=%s", key)
except Exception as exc: # noqa: BLE001
log.error("event=orphan_delete_failed key=%s type=%s",
key, type(exc).__name__)
# staging/ — удаляем только старое И не принадлежащее активной задаче.
# Ключ: staging/<chat_id>/<attachment_id>/original
for obj in s3.list_objects("staging/"):
key = obj["Key"]
parts = key.split("/")
# Ожидаем минимум 4 части: staging / chat_id / attachment_id / original
if len(parts) < 4 or parts[0] != "staging":
continue
attachment_id = parts[2]
if attachment_id in active_ids:
# Задача ещё в queued/processing — staging нужен worker'у.
continue
lm = obj.get("LastModified")
if lm is None:
continue
if lm.tzinfo is None:
lm = lm.replace(tzinfo=timezone.utc)
if lm > grace_cutoff:
continue
try:
s3.delete_object(key)
staging_deleted += 1
except Exception:
pass
# missing
for key in known:
if s3.head_object(key) is None:
missing += 1
log.warning("event=missing_object key=%s", key)
# зависшие processing → queued
try:
recovered = storage.recover_stuck_processing()
if recovered:
log.info("event=stuck_processing_recovered count=%s", recovered)
except Exception as exc: # noqa: BLE001
log.error("event=recover_stuck_failed type=%s", type(exc).__name__)
return {
"scanned": scanned,
"orphan_deleted": orphan_deleted,
"missing": missing,
"known": len(known),
"staging_deleted": staging_deleted,
}
def main() -> None:
init_pool()
interval = _interval()
log.info("event=cleanup_start interval_s=%s grace_s=%s", interval, _grace())
try:
while True:
try:
stats = _run_once()
log.info(
"event=cleanup_pass scanned=%s known=%s orphan_deleted=%s"
" missing=%s staging_deleted=%s",
stats["scanned"], stats["known"], stats["orphan_deleted"],
stats["missing"], stats["staging_deleted"],
)
except Exception as exc: # noqa: BLE001
log.error("event=cleanup_failed type=%s", type(exc).__name__)
time.sleep(interval)
finally:
close_pool()
if __name__ == "__main__":
main()
+143
View File
@@ -0,0 +1,143 @@
"""Хранилище сессий. Redis (основной режим, lab2 level 4) или PostgreSQL (fallback).
Ключевое: при недоступности Redis поднимаем RuntimeError("redis_unavailable"),
чтобы main.py отдал 503 unavailable, а не 500 internal_error.
"""
import json
import os
import secrets
from datetime import datetime, timedelta, timezone
from typing import Optional
from uuid import UUID
from ..models.schemas import User
from . import db as _db
try:
import redis as _redis
except ImportError: # pragma: no cover
_redis = None
_SESSION_PREFIX = "sess:"
def now_utc() -> datetime:
return datetime.now(timezone.utc)
def _store() -> str:
return os.environ.get("SESSION_STORE", "redis").strip().lower()
def _redis_client():
"""Возвращает подключённый клиент Redis или поднимает RuntimeError.
RuntimeError("redis_unavailable: ...") ловит main.runtime_error_handler → 503.
"""
if _redis is None:
raise RuntimeError("redis_unavailable: redis library is not installed")
url = os.environ.get("REDIS_URL", "redis://redis:6379/0")
try:
client = _redis.from_url(
url,
decode_responses=True,
socket_timeout=2.0,
socket_connect_timeout=2.0,
)
client.ping()
return client
except _redis.RedisError as exc:
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
def _user_key(user_id: UUID) -> str:
return str(user_id)
def create_session(user_id: UUID, ttl_seconds: int) -> tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = now_utc() + timedelta(seconds=ttl_seconds)
if _store() == "redis":
r = _redis_client()
payload = json.dumps({"user_id": _user_key(user_id), "expires_at": expires_at.isoformat()})
try:
# SETEX: TTL — сама суть Redis-сессий
r.setex(_SESSION_PREFIX + token, ttl_seconds, payload)
except _redis.RedisError as exc:
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
else:
with _db.connection() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO sessions (token, user_id, expires_at) VALUES (%s, %s, %s)",
(token, user_id, expires_at),
)
conn.commit()
return token, expires_at
def get_session_user(token: str) -> Optional[User]:
if _store() == "redis":
r = _redis_client()
try:
raw = r.get(_SESSION_PREFIX + token)
except _redis.RedisError as exc:
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
if raw is None:
return None
try:
data = json.loads(raw)
user_id = UUID(data["user_id"])
expires_at = datetime.fromisoformat(data["expires_at"])
except Exception:
return None
if expires_at <= now_utc():
try:
r.delete(_SESSION_PREFIX + token)
except _redis.RedisError:
pass
return None
return _load_user(user_id)
# postgres fallback
with _db.connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
" FROM sessions s JOIN users u ON u.id = s.user_id"
" WHERE s.token = %s",
(token,),
)
row = cur.fetchone()
if row is None:
return None
if row[4] <= now_utc():
cur.execute("DELETE FROM sessions WHERE token = %s", (token,))
conn.commit()
return None
return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3])
def revoke_session(token: str) -> None:
if _store() == "redis":
r = _redis_client()
try:
r.delete(_SESSION_PREFIX + token)
except _redis.RedisError as exc:
raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc
return
with _db.connection() as conn:
with conn.cursor() as cur:
cur.execute("DELETE FROM sessions WHERE token = %s", (token,))
conn.commit()
def _load_user(user_id: UUID) -> Optional[User]:
with _db.connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
)
row = cur.fetchone()
return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3]) if row else None
+160
View File
@@ -0,0 +1,160 @@
"""Единая точка инициализации OpenTelemetry для API и worker.
- OTLP/HTTP → Collector (otel-collector:4318).
- resource: service.name/version/instance.id/deployment.environment.name=lab.
- LoggingFilter подставляет trace_id/span_id в каждую запись, чтобы они
попадали и в stdout (видно глазами), и в Loki (в тексте строки) — это
требование lab6 level4 (связь лога и trace).
"""
import logging
import os
import uuid
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
_INITIALIZED = False
INSTANCE_ID = os.environ.get("INSTANCE_ID") or uuid.uuid4().hex[:12]
def _endpoint() -> str:
return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
def _resource(service_name: str, service_version: str) -> Resource:
return Resource.create(
{
"service.name": service_name,
"service.version": service_version,
"service.instance.id": INSTANCE_ID,
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
}
)
def _build_trace_provider(resource: Resource) -> TracerProvider:
provider = TracerProvider(resource=resource)
exporter = OTLPSpanExporter(endpoint=_endpoint() + "/v1/traces", timeout=5)
provider.add_span_processor(
BatchSpanProcessor(exporter, max_queue_size=2048,
max_export_batch_size=512,
schedule_delay_millis=2000)
)
return provider
def _build_meter_provider(resource: Resource) -> MeterProvider:
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=_endpoint() + "/v1/metrics", timeout=5),
export_interval_millis=5000,
)
return MeterProvider(resource=resource, metric_readers=[reader])
def _build_logger_provider(resource: Resource) -> LoggerProvider:
provider = LoggerProvider(resource=resource)
provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=_endpoint() + "/v1/logs", timeout=5),
max_queue_size=2048,
max_export_batch_size=512,
schedule_delay_millis=2000,
)
)
return provider
class TraceContextFilter(logging.Filter):
"""Подставляет record.trace_id / record.span_id из активного span.
Если активного span нет — пишет "0"*32 и "0"*16, чтобы лог-формат
оставался валидным (не падал с KeyError). Так же OTel сам поступает.
"""
def filter(self, record: logging.LogRecord) -> bool:
span = trace.get_current_span()
span_ctx = span.get_span_context() if span else None
if span_ctx is not None and span_ctx.is_valid:
record.trace_id = format(span_ctx.trace_id, "032x")
record.span_id = format(span_ctx.span_id, "016x")
else:
record.trace_id = "0" * 32
record.span_id = "0" * 16
return True
_LOG_FORMAT = (
"%(asctime)s %(levelname)s %(name)s "
"trace_id=%(trace_id)s span_id=%(span_id)s %(message)s"
)
def install_log_format() -> None:
"""Единый формат логов API/worker/outbox.
Поля trace_id/span_id гарантированно существуют благодаря
TraceContextFilter. Их наличие обеспечивает уровень 4 (корреляция
логов и trace) и делает Loki-запросы `|= "trace_id="` рабочими.
"""
handler = logging.StreamHandler()
handler.addFilter(TraceContextFilter())
handler.setFormatter(logging.Formatter(_LOG_FORMAT))
root = logging.getLogger()
root.handlers = [handler]
root.setLevel(logging.INFO)
def init_telemetry(service_name: str, service_version: str = "1.6.0") -> None:
"""Идемпотентная инициализация OTel для процесса."""
global _INITIALIZED
if _INITIALIZED:
return
# Сначала — единый формат логов и фильтр trace_id.
install_log_format()
resource = _resource(service_name, service_version)
trace.set_tracer_provider(_build_trace_provider(resource))
metrics.set_meter_provider(_build_meter_provider(resource))
logger_provider = _build_logger_provider(resource)
set_logger_provider(logger_provider)
# OTel handler шлёт те же записи в Collector → Loki. Он тоже видит
# trace_id из record, потому что span-контекст един.
#
# ВАЖНО: LoggingHandler.emit() формирует тело через self.format(record),
# где self.formatter наследуется от logging.Handler. Без явного
# formatter'а тело будет голым сообщением (msg % args) — без
# trace_id=/span_id=. Это ломало поиск в Loki `|= "trace_id="`.
# Поэтому задаём тот же формат и навешиваем TraceContextFilter
# непосредственно на OTel-хэндлер: фильтр должен отработать ДО emit,
# чтобы record.trace_id существовал на момент форматирования.
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otel_handler.addFilter(TraceContextFilter())
otel_handler.setFormatter(logging.Formatter(_LOG_FORMAT))
logging.getLogger().addHandler(otel_handler)
_INITIALIZED = True
logging.getLogger("telemetry").info(
"event=otel_init service=%s instance_id=%s endpoint=%s",
service_name, INSTANCE_ID, _endpoint(),
)
def tracer(name: str = "app"):
return trace.get_tracer(name)
def meter(name: str = "app"):
return metrics.get_meter(name)
+405
View File
@@ -0,0 +1,405 @@
"""Worker lab5/lab6: обработка вложений из RabbitMQ с трейсингом.
- at-least-once, идемпотентно, retry через TTL-очередь, DLQ.
- OTel: consumer span на каждый job, извлечение trace context из заголовков
сообщения (W3C traceparent/tracestate), child spans на S3/DB операции.
- Метрики: jobs_processed_total{outcome}, job_processing_duration_seconds.
"""
import io
import json
import logging
import os
import signal
import sys
import time
import pika
from PIL import Image, ImageOps
from opentelemetry import context as otel_context, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from .storage import s3
from .storage.db import init_pool, close_pool
from .storage.postgres import storage
from .storage.rabbit import (
QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology,
publish_retry, publish_dlq,
)
from .telemetry import init_telemetry, meter
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
log = logging.getLogger("worker")
PIPELINE_VERSION = 1
LEASE_SECONDS = 60
THUMB_SIZE = (256, 256)
CROP_SIZE = (128, 128)
OPT_SIZE = (1280, 1280)
OPT_QUALITY = 82
_stop = False
_propagator = TraceContextTextMapPropagator()
# OTel инициализация — до первого сообщения
init_telemetry("messenger-worker", "1.6.0")
_tracer = trace.get_tracer("worker")
_meter = meter("messenger-worker")
_jobs_counter = _meter.create_counter(
"jobs_processed_total", description="Jobs processed by outcome", unit="1",
)
# Явные границы бакетов в СЕКУНДАХ: job-обработка (decode + resize + 3×S3 PUT
# + DB) обычно от десятков мс до единиц секунд. Дефолтные экспоненциальные
# бакеты слишком грубы для p95/p99 на этой шкале.
_JOB_DURATION_BUCKETS = [
0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5,
0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0,
]
_jobs_hist = _meter.create_histogram(
"job_processing_duration_seconds",
description="Job processing duration",
unit="s",
explicit_bucket_boundaries_advisory=_JOB_DURATION_BUCKETS,
)
# --- span helpers: дочерние операции S3/DB ---
#
# ВАЖНО: атрибуты, содержащие точку (`db.operation`, `aws.s3.key` и т.п.),
# нельзя передавать как kwargs — Python-идентификатор с точкой невозможен.
# Поэтому оба хелпера принимают dict атрибутов.
def _s3_span(name: str, attrs: dict):
"""Context manager для S3-операции (CLIENT span внутри job.consume)."""
return _tracer.start_as_current_span(
name,
kind=SpanKind.CLIENT,
attributes={"messaging.system": "s3", **attrs},
)
def _db_span(name: str, attrs: dict):
"""Context manager для DB-операции (CLIENT span внутри job.consume)."""
return _tracer.start_as_current_span(
name,
kind=SpanKind.CLIENT,
attributes={"db.system": "postgresql", **attrs},
)
def _handle_sigterm(signum, frame):
global _stop
log.info("event=sigterm_received signal=%s", signum)
_stop = True
def _now_iso() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def _to_rgb_white(img: Image.Image) -> Image.Image:
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
img = img.convert("RGBA")
bg = Image.new("RGB", img.size, (255, 255, 255))
bg.paste(img, mask=img.split()[-1])
return bg
if img.mode != "RGB":
return img.convert("RGB")
return img
def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]:
w, h = img.size
scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0)
nw = max(1, int(w * scale))
nh = max(1, int(h * scale))
thumb = img.resize((nw, nh), Image.LANCZOS)
buf = io.BytesIO()
thumb.save(buf, format="JPEG", quality=88, optimize=True)
return thumb, buf.getvalue()
def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]:
w, h = img.size
side = min(w, h)
left = (w - side) // 2
top = (h - side) // 2
sq = img.crop((left, top, left + side, top + side))
crop = sq.resize(CROP_SIZE, Image.LANCZOS)
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=88, optimize=True)
return crop, buf.getvalue()
def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]:
w, h = img.size
scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0)
nw = max(1, int(w * scale))
nh = max(1, int(h * scale))
opt = img.resize((nw, nh), Image.LANCZOS)
buf = io.BytesIO()
opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
return opt, buf.getvalue()
def _sha256(data: bytes) -> str:
import hashlib
return hashlib.sha256(data).hexdigest()
def _s3_put_variant(chat_id, att_id, name: str, data: bytes, ctype: str) -> str:
"""Общий путь: S3 PUT варианта внутри CLIENT span."""
key = s3.variant_key(chat_id, att_id, name)
with _s3_span("s3.put_object", {
"aws.s3.key": key,
"s3.operation": "put_object",
"s3.variant": name,
"s3.content_type": ctype,
"s3.size_bytes": len(data),
}):
s3.put_bytes(key, data, ctype)
return key
def process_job(payload: dict) -> None:
"""Обработка одного job. Идемпотентна через finalize_ready.
Внутри активного job.consume (CONSUMER) span создаются дочерние CLIENT
spans для S3 и DB, чтобы разложить задержку обработки по зависимостям.
"""
attachment_id = payload["attachment_id"]
pipeline_version = payload.get("pipeline_version", 1)
job_id = payload["job_id"]
log.info("event=job_received job_id=%s attachment_id=%s attempt=%s",
job_id, attachment_id, payload.get("attempt", 1))
with _db_span("db.claim_processing", {
"db.operation": "UPDATE",
"attachment_id": str(attachment_id),
}):
claimed = storage.claim_processing(attachment_id, LEASE_SECONDS)
if not claimed:
log.info("event=job_skipped_already_done attachment_id=%s", attachment_id)
return
pair = storage.get_attachment_with_key(attachment_id)
if pair is None:
log.warning("event=attachment_missing attachment_id=%s", attachment_id)
return
att, staging_key = pair
if att.status == "ready":
log.info("event=already_ready attachment_id=%s", attachment_id)
return
try:
with _s3_span("s3.get_object", {
"aws.s3.key": staging_key,
"s3.operation": "get_object",
"s3.variant": "staging",
}):
raw, ctype = s3.get_object(staging_key)
except Exception as exc: # noqa: BLE001
raise RuntimeError("staging_unavailable") from exc
try:
img = Image.open(io.BytesIO(raw))
img.load()
except Exception as exc: # noqa: BLE001
with _db_span("db.finalize_failed", {
"db.operation": "UPDATE",
"error_code": "invalid_image",
}):
storage.finalize_failed(attachment_id, "invalid_image")
log.warning("event=invalid_image attachment_id=%s type=%s",
attachment_id, type(exc).__name__)
return
img_rgb = _to_rgb_white(img)
# Финал: S3 PUT оригинала
final_key = s3.original_key(att.chat_id, att.id)
with _s3_span("s3.put_object", {
"aws.s3.key": final_key,
"s3.operation": "put_object",
"s3.variant": "original",
"s3.content_type": att.content_type,
"s3.size_bytes": len(raw),
}):
s3.put_bytes(final_key, raw, att.content_type)
variants = [{
"name": "original",
"object_key": final_key,
"content_type": att.content_type,
"size_bytes": len(raw),
"width": img.width,
"height": img.height,
"sha256": _sha256(raw),
}]
# Thumbnail
_, thumb_bytes = _make_thumbnail(img_rgb)
thumb_key = _s3_put_variant(att.chat_id, att.id, "thumbnail", thumb_bytes, "image/jpeg")
tw, th = Image.open(io.BytesIO(thumb_bytes)).size
variants.append({
"name": "thumbnail", "object_key": thumb_key,
"content_type": "image/jpeg", "size_bytes": len(thumb_bytes),
"width": tw, "height": th, "sha256": _sha256(thumb_bytes),
})
# Crop
_, crop_bytes = _make_crop(img_rgb)
crop_key = _s3_put_variant(att.chat_id, att.id, "crop", crop_bytes, "image/jpeg")
cw, ch = Image.open(io.BytesIO(crop_bytes)).size
variants.append({
"name": "crop", "object_key": crop_key,
"content_type": "image/jpeg", "size_bytes": len(crop_bytes),
"width": cw, "height": ch, "sha256": _sha256(crop_bytes),
})
# Optimized
_, opt_bytes = _make_optimized(img_rgb)
opt_key = _s3_put_variant(att.chat_id, att.id, "optimized", opt_bytes, "image/webp")
ow, oh = Image.open(io.BytesIO(opt_bytes)).size
variants.append({
"name": "optimized", "object_key": opt_key,
"content_type": "image/webp", "size_bytes": len(opt_bytes),
"width": ow, "height": oh, "sha256": _sha256(opt_bytes),
})
with _db_span("db.finalize_ready", {
"db.operation": "UPDATE",
"pipeline_version": pipeline_version,
"variants_count": len(variants),
}):
storage.finalize_ready(attachment_id, pipeline_version, variants)
# Cleanup staging — best-effort, не критичен для результата.
try:
with _s3_span("s3.delete_object", {
"aws.s3.key": staging_key,
"s3.operation": "delete_object",
"s3.variant": "staging",
}):
s3.delete_object(staging_key)
except Exception:
pass
log.info("event=job_ready attachment_id=%s job_id=%s variants=%d",
attachment_id, job_id, len(variants))
def _extract_trace_context(properties) -> otel_context.Context:
"""Извлекает W3C traceparent/tracestate из headers. Ошибка → пустой контекст."""
try:
headers = dict(properties.headers or {})
carrier = {
k.lower(): str(v)
for k, v in headers.items()
if k.lower() in ("traceparent", "tracestate")
}
if not carrier:
return otel_context.Context()
return _propagator.extract(carrier=carrier)
except Exception as exc: # noqa: BLE001
log.warning("event=trace_extract_failed type=%s", type(exc).__name__)
return otel_context.Context()
def _on_message(ch, method, properties, body):
"""Callback consumer. ack/nack — явно. Оборачивает обработку в CONSUMER span."""
parent_ctx = _extract_trace_context(properties)
started = time.monotonic()
with _tracer.start_as_current_span(
"job.consume", context=parent_ctx, kind=SpanKind.CONSUMER,
attributes={"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE},
) as span:
try:
payload = json.loads(body)
except Exception:
log.error("event=invalid_job_json")
_jobs_counter.add(1, {"job.type": "unknown", "outcome": "invalid"})
ch.basic_nack(method.delivery_tag, requeue=False)
return
headers = dict(properties.headers or {})
retry_count = int(headers.get("x-retry-count", 0))
payload["attempt"] = retry_count + 1
payload["_headers"] = headers
span.set_attribute("job.id", str(payload.get("job_id", "")))
span.set_attribute("job.type", str(payload.get("type", "unknown")))
span.set_attribute("job.attempt", retry_count + 1)
try:
process_job(payload)
duration = time.monotonic() - started
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "success"})
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "success"})
span.set_status(Status(StatusCode.OK))
ch.basic_ack(method.delivery_tag)
except Exception as exc: # noqa: BLE001
duration = time.monotonic() - started
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
log.warning("event=job_retry attachment_id=%s attempt=%d type=%s",
payload.get("attachment_id"), retry_count + 1, type(exc).__name__)
if retry_count + 1 >= MAX_ATTEMPTS:
try:
with _db_span("db.finalize_failed", {
"db.operation": "UPDATE",
"error_code": "retry_exhausted",
}):
storage.finalize_failed(payload["attachment_id"], "retry_exhausted")
except Exception:
pass
try:
publish_dlq({k: v for k, v in payload.items() if k != "_headers"})
except Exception:
pass
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "failed"})
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "failed"})
ch.basic_ack(method.delivery_tag)
return
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "retry"})
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "retry"})
try:
publish_retry(payload, retry_count + 1)
ch.basic_ack(method.delivery_tag)
except Exception:
ch.basic_nack(method.delivery_tag, requeue=True)
def main() -> None:
signal.signal(signal.SIGTERM, _handle_sigterm)
init_pool()
log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS)
while not _stop:
try:
conn = connect()
ch = conn.channel()
setup_topology(ch)
ch.basic_qos(prefetch_count=1)
ch.basic_consume(queue=QUEUE, on_message_callback=_on_message)
log.info("event=consumer_started")
while not _stop and conn.is_open:
conn.process_data_events(time_limit=1)
ch.stop_consuming()
conn.close()
except Exception as exc: # noqa: BLE001
log.error("event=worker_loop_error type=%s", type(exc).__name__)
time.sleep(3)
close_pool()
log.info("event=worker_stop")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Единая команда запуска собственных тестов.
Запускает: test_negative.py, test_concurrency.py.
Публичный smoke запускается отдельно."""
import subprocess
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
def run_one(script, base_url):
print("=== %s ===" % script.name)
proc = subprocess.run(
[sys.executable, str(script), "--base-url", base_url],
cwd=str(HERE),
)
if proc.returncode != 0:
print("FAIL: %s" % script.name)
return False
return True
def main():
import argparse
p = argparse.ArgumentParser(description="Lab1 student tests (level 4)")
p.add_argument("--base-url", default="http://localhost:8080")
args = p.parse_args()
ok = True
for script in ("lab1_test_negative.py", "lab1_test_concurrency.py"):
if not run_one(HERE / script, args.base_url):
ok = False
if not ok:
raise SystemExit(1)
print("PASS: all student tests (level 4) passed")
if __name__ == "__main__":
main()
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env python3
"""Конкурентный тест: 100 сообщений от 10 клиентов -> ровно 100 уникальных.
Проверяет защиту in-memory структур от гонок (RLock)."""
import sys
import threading
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run # noqa: E402
TOTAL_MESSAGES = 100
CLIENTS = 10
def main():
args = arguments("Lab1 concurrency safety")
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"]
# Каждый «клиент» — свой actor (alice/bob) для реалистичной конкуренции
actors = [alice, bob]
barrier = threading.Barrier(CLIENTS)
def worker(worker_id):
local = Client(args.base_url, args.ca_file, timeout=15)
actor = actors[worker_id % len(actors)]
barrier.wait() # синхронный старт всех потоков
produced = []
for i in range(TOTAL_MESSAGES // CLIENTS):
text = "c-%02d-%02d-%s" % (worker_id, i, uuid.uuid4().hex[:8])
msg = local.message(actor, chat, text)["json"]
produced.append(msg)
return produced
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
results = list(pool.map(worker, range(CLIENTS)))
all_ids = [m["id"] for chunk in results for m in chunk]
check(len(all_ids) == TOTAL_MESSAGES,
"expected %d message ids, got %d" % (TOTAL_MESSAGES, len(all_ids)))
check(len(set(all_ids)) == TOTAL_MESSAGES,
"duplicate message ids under concurrency: %d unique" % len(set(all_ids)))
# Читаем всю историю через пагинацию
collected = []
cursor = None
from urllib.parse import quote
for _ in range(50):
q = path + "/messages?limit=100"
if cursor:
q += "&cursor=" + quote(cursor, safe="")
page = c.call("GET", q, actor=alice, schema="MessagePage")["json"]
collected.extend(m["id"] for m in page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
check(len(collected) == TOTAL_MESSAGES,
"stored messages: %d, expected %d" % (len(collected), TOTAL_MESSAGES))
check(len(set(collected)) == TOTAL_MESSAGES,
"stored messages contain duplicates: %d unique" % len(set(collected)))
print("PASS: lab1 concurrency: %d messages from %d clients, "
"%d unique stored and readable" % (TOTAL_MESSAGES, CLIENTS, len(set(collected))))
if __name__ == "__main__":
run(main)
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Трек 1 (оценка 5). Предел in-memory архитектуры.
Гипотеза: на больших историях список сообщений при чтении каждый раз
копируется и сортируется под RLock, что ухудшает p95/p99 при росте
истории. Убираем сортировку на чтении (append сохраняет порядок) и
сравниваем "до/после" при тех же лимитах.
Эксперимент: >=3 размера истории (H) x >=3 уровня конкуренции (C).
Метрики: successful RPS, доля ошибок, p50/p95/p99 только успешных
запросов, RSS процесса API (читается через docker stats или /proc).
Скрипт не меняет API; использует те же публичные клиенты из tests/.
Stdlib only, Python 3.9+.
"""
import json
import math
import statistics
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
# --- профиль эксперимента -------------------------------------------------
HISTORY_SIZES = [100, 1000, 5000] # >=3 размера истории
CONCURRENCY_LEVELS = [1, 4, 16] # >=3 уровня конкуренции
DURATION_SECONDS = 10 # на каждую точку (H, C)
API_CONTAINER = "mtusi-messenger-lab1" # имя контейнера для docker stats
def rss_mb(container: str):
"""RSS процесса API в МБ через docker stats --no-stream."""
try:
out = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container],
capture_output=True, text=True, timeout=10,
)
if out.returncode != 0:
return None
# Пример: "120.5MiB / 1GiB"
used = out.stdout.strip().split("/")[0].strip()
if used.endswith("MiB"):
return round(float(used[:-3]), 2)
if used.endswith("GiB"):
return round(float(used[:-3]) * 1024, 2)
if used.endswith("KiB"):
return round(float(used[:-3]) / 1024, 2)
except Exception:
return None
return None
def seed_history(c, actor, chat, count, workers=16):
"""Заполняет историю до count сообщений параллельно."""
from concurrent.futures import ThreadPoolExecutor
def one(i):
local = Client(c.base, timeout=15)
local.message(actor, chat, "seed-%06d-%s" % (i, uuid.uuid4().hex[:6]))
with ThreadPoolExecutor(max_workers=workers) as pool:
list(pool.map(one, range(count)))
def measure_read_point(c, chat, actor, duration, concurrency):
"""Закрытый контур: параллельно читаем GET /messages?limit=100."""
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
barrier = threading.Barrier(concurrency)
latencies, errors = [], 0
errors_lock = threading.Lock()
def worker(_):
nonlocal errors
local = Client(c.base, timeout=15)
barrier.wait()
deadline = time.monotonic() + duration
local_lat = []
while time.monotonic() < deadline:
start = time.monotonic()
try:
local.call("GET", path, actor=actor, schema="MessagePage")
local_lat.append(time.monotonic() - start)
except Failure:
with errors_lock:
errors += 1
return local_lat
started = time.monotonic()
with ThreadPoolExecutor(max_workers=concurrency) as pool:
results = list(pool.map(worker, range(concurrency)))
elapsed = time.monotonic() - started
for chunk in results:
latencies.extend(chunk)
latencies.sort()
def pct(p):
if not latencies:
return None
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
total = len(latencies) + errors
return {
"history_size": None, # заполнит вызывающий
"concurrency": concurrency,
"duration_s": round(elapsed, 3),
"successful": len(latencies),
"errors": errors,
"error_ratio": round(errors / total, 6) if total else None,
"rps": round(len(latencies) / elapsed, 3),
"latency_ms": {"p50": pct(.50), "p95": pct(.95), "p99": pct(.99)},
}
def main():
args = arguments("Трек 1: предел in-memory архитектуры")
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
report = {"lab": 1, "track": 1, "profile": {
"history_sizes": HISTORY_SIZES,
"concurrency_levels": CONCURRENCY_LEVELS,
"duration_seconds": DURATION_SECONDS,
"note": "closed-loop; клиент ограничивает throughput; RSS — docker stats",
}, "points": []}
for h in HISTORY_SIZES:
# Досеиваем историю до h (bootstrap уже создал часть).
# Считаем число сообщений в истории.
# Предыдущая версия: limit=1 - возвращает только одну страницу и далее считает её длину страницы за длину всей истории,
# Текущая версия: limit=100 - делаем полный обход истории по курсору => реальное число сообщений
have = 0
cursor = None
from urllib.parse import quote
while True:
q = path + "?limit=100"
if cursor:
q += "&cursor=" + quote(cursor, safe="")
page = c.call("GET", q, actor=alice, schema="MessagePage")["json"]
have += len(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
if have < h:
seed_history(c, alice, chat, h - have)
for con in CONCURRENCY_LEVELS:
point = measure_read_point(c, chat, alice, DURATION_SECONDS, con)
point["history_size"] = h
point["rss_mb"] = rss_mb(API_CONTAINER)
report["points"].append(point)
print(json.dumps(point, ensure_ascii=False))
out = Path("evidence/lab1_track1_load.json")
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print("PASS: track1 load matrix saved to", out)
if __name__ == "__main__":
# Дополнительный CLI: --container для имени Docker-контейнера.
import argparse
p = argparse.ArgumentParser(add_help=False)
p.add_argument("--container", default=API_CONTAINER)
extra, _ = p.parse_known_args()
# Прокидываем в namespace аргументов arguments() через env-подобный трюк.
sys.argv = [a for a in sys.argv if not a.startswith("--container")]
import os
os.environ.setdefault("TRACK1_CONTAINER", extra.container)
def _bind_container(ns):
ns.container = os.environ.get("TRACK1_CONTAINER", API_CONTAINER)
return ns
# Патчим arguments, чтобы добавить --container.
import client as _client # noqa: E402
_orig_arguments = _client.arguments
def _arguments(description, configure=None):
def cfg(parser):
if configure:
configure(parser)
parser.add_argument("--container", default=API_CONTAINER)
return _orig_arguments(description, cfg)
_client.arguments = _arguments
run(main)
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""Негативные сценарии lab1: неверный JSON, чужой чат, пустое сообщение,
дубликат username, обход нескольких страниц. Stdlib only, Python 3.9+."""
import sys
import urllib.error
import urllib.request
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def _raw_call(base_url, method, path, body=None, headers=None):
"""Сырой запрос для проверки malformed JSON (client.py всегда сериализует)."""
url = base_url.rstrip("/") + path
data = body if isinstance(body, (bytes, type(None))) else body.encode("utf-8")
req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
return resp.status, resp.read()
except urllib.error.HTTPError as e:
return e.code, e.read()
def main():
args = arguments("Lab1 negative scenarios")
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"]
# 1. Дубликат username -> 409
c.call("POST", "/api/v1/users", 409, body=alice["registration"])
# 2. Неверный JSON (malformed) -> 400
status, _ = _raw_call(
c.base,
"POST",
"/api/v1/users",
body=b"{not json at all",
headers={"Content-Type": "application/json"},
)
check(status == 400, "malformed JSON must be 400, got %s" % status)
# 3. Лишние поля -> 400
c.call(
"POST",
"/api/v1/users",
400,
body={"username": "extra_user_x", "display_name": "X", "unexpected": True},
)
# 4. Неверный тип -> 400
c.call("POST", "/api/v1/users", 400, body={"username": 123, "display_name": "X"})
# 5. Пустое / пробельное сообщение -> 400
c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""})
c.call("POST", path + "/messages", 400, actor=alice, body={"text": " "})
# 6. Чужой чат -> 404 (не 403)
c.call("GET", path, 404, actor=eve)
c.call("GET", path + "/messages", 404, actor=eve)
c.call("POST", path + "/messages", 404, actor=eve, body={"text": "чужое"})
# 7. Пагинация: обход нескольких страниц без потерь и дублей
texts = ["p-%02d" % i for i in range(7)]
for t in texts:
c.message(alice, chat, t)
collected = []
cursor = None
seen_cursors = set()
for _ in range(20): # предохранитель от бесконечного цикла
q = path + "/messages?limit=2"
if cursor:
from urllib.parse import quote
q += "&cursor=" + quote(cursor, safe="")
page = c.call("GET", q, actor=bob, schema="MessagePage")["json"]
collected.extend(m["text"] for m in page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
check(cursor not in seen_cursors, "cursor loop detected")
seen_cursors.add(cursor)
check(len(collected) == len(texts), "pagination lost messages: %d != %d" % (len(collected), len(texts)))
check(len(set(collected)) == len(collected), "pagination duplicated messages")
check(collected == texts, "pagination order/contents mismatch")
# 8. Некорректный limit -> 400
c.call("GET", path + "/messages?limit=0", 400, actor=alice)
c.call("GET", path + "/messages?limit=101", 400, actor=alice)
# 9. Неизвестный X-User-Id -> 401
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": str(uuid.uuid4())})
print("PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, "
"wrong type, empty text, foreign chat, pagination, bad limit, unknown user)")
if __name__ == "__main__":
run(main)
+587
View File
@@ -0,0 +1,587 @@
#!/usr/bin/env python3
"""Уровень 4 lab2: expiry TTL, конкурентная регистрация, EXPLAIN, недоступность Redis/PostgreSQL.
Перед запуском:
1) в .env выставить SESSION_TTL_SECONDS=5
2) docker compose --env-file .env up -d --force-recreate api
3) docker exec mtuci-redis redis-cli flushdb
4) python student-tests/test_lab2_level4.py --base-url http://localhost:8080
5) перед smoke/persistence вернуть SESSION_TTL_SECONDS=3600 и снова up -d --force-recreate api
Порядок тестов: тяжёлые по данным и конкурентные — ДО expiry-теста,
потому что при коротком TTL любой долгий шаг съедает окно жизни сессии.
Expiry-тест запускается последним и использует свежие сессии.
test_explain не зависит от TTL вообще: он сидирует данные напрямую
через psql и проверяет план SQL-запроса, а не HTTP-слой.
test_chat_rollback проверяет транзакционную границу create_chat: он
запускает патч-скрипт ВНУТРИ контейнера API (там есть psycopg и
DATABASE_URL), monkey-patch-ит psycopg.Cursor.executemany так, чтобы
INSERT в chat_members упал ПОСЛЕ успешного INSERT в chats. Затем через
psql проверяется, что ни chat, ни chat_members не сохранились.
"""
import base64
import datetime as dt
import os
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name).
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default")
# Имя контейнера API (зафиксировано в compose.yaml container_name).
API_CONTAINER = os.environ.get("API_CONTAINER", "mtuci-messenger-lab2")
# ---------- 1. Конкурентная регистрация одинакового username ----------
def test_concurrent_registration(c):
"""Один и тот же username с 8 потоков — ровно одна 201, остальные 409.
Проверяем ФАКТИЧЕСКИЕ HTTP-коды, а не факт исключения: timeout, 500
или неверный формат ответа должны проваливать тест, а не выдаваться
за ожидаемый конфликт. Дополнительно сверяем, что в БД создана
ровно одна строка пользователя.
"""
username = "race_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
body = {"username": username, "display_name": "Race", "password": password}
results = []
lock = threading.Lock()
def one(_):
local = Client(c.base, timeout=10)
try:
r = local.call("POST", "/api/v1/users", (201, 409),
body=body, schema="User" if True else None)
# schema валидируется только для 2xx; при 409 приходит Error.
status = r["status"]
except Failure as exc:
# Любая иная ошибка (timeout, 500, bad schema) — это не 409.
status = "failure:%s" % exc
with lock:
results.append(status)
with ThreadPoolExecutor(max_workers=8) as pool:
list(pool.map(one, range(8)))
ok = [r for r in results if r == 201]
conflict = [r for r in results if r == 409]
bad = [r for r in results if r not in (201, 409)]
check(not bad, "unexpected results under concurrent registration: %s" % results)
check(len(ok) == 1, "expected exactly one 201, got %s" % results)
check(len(conflict) == 7, "expected seven 409, got %s" % results)
# В БД должна быть ровно одна строка с этим username.
dsn = _read_dsn()
rows = _scalar(dsn, "SELECT count(*) FROM users WHERE username = '%s';" % username)
check(int(rows) == 1,
"ожидалась ровно одна строка users для %s, получено %s" % (username, rows))
# ---------- 2. EXPLAIN на реальной БД ----------
def _read_dsn() -> str:
dsn = os.environ.get("DATABASE_URL")
if not dsn:
env_file = Path(__file__).resolve().parents[1] / ".env"
if env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line.startswith("DATABASE_URL="):
dsn = line.split("=", 1)[1].strip()
break
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
# Postgres доступен только внутри compose-сети; одноразовые psql-контейнеры
# запускаются в этой же сети, поэтому имя хоста postgres резолвится как есть.
return dsn
def _psql(dsn_in_container: str, sql: str) -> str:
r = subprocess.run(
[
"docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"postgres:18-alpine",
"psql", dsn_in_container, "-tA", "-c", sql,
],
capture_output=True, text=True, timeout=120,
)
return r.stdout + r.stderr
def _scalar(dsn: str, sql: str) -> str:
r = subprocess.run(
[
"docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"postgres:18-alpine",
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1", "-c", sql,
],
capture_output=True, text=True, timeout=120,
)
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
return r.stdout.strip().splitlines()[-1]
def test_explain(c):
"""EXPLAIN (ANALYZE, BUFFERS) истории сообщений на >=10 000 строк.
Методичка: индекс истории обоснован запросом и EXPLAIN на >=10 000
сообщениях. На пустой таблице планировщик выбирает Seq Scan, поэтому
тест сам доводит messages до >=10 000 строк и делает ANALYZE.
Данные сидируются напрямую через psql, без API: тест не зависит от
SESSION_TTL_SECONDS и скорости scrypt. Это корректно, потому что
EXPLAIN проверяет план SQL-запроса, а не HTTP-слой.
"""
dsn_in_container = _read_dsn()
# Смотрим, сколько строк уже есть.
have_raw = _psql(dsn_in_container, "SELECT count(*) FROM messages;")
have = int(have_raw.strip().splitlines()[-1])
print(" в таблице messages сейчас: %d строк" % have)
target_total = 10_000
if have < target_total:
need = target_total - have
print(" сидируем %d сообщений напрямую через psql ..." % need)
# Один чат: берём существующий или создаём минимальный.
# gen_random_uuid() доступен в PG13+ без расширений.
seed_sql = """
DO $$
DECLARE
v_user UUID;
v_chat UUID;
BEGIN
SELECT id INTO v_user FROM users LIMIT 1;
IF v_user IS NULL THEN
v_user := gen_random_uuid();
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed',
'scrypt$16384$8$1$00$00', now());
END IF;
SELECT id INTO v_chat FROM chats LIMIT 1;
IF v_chat IS NULL THEN
v_chat := gen_random_uuid();
INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'Seed chat', now());
INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user);
END IF;
-- Доливаем ровно need строк в этот чат.
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), v_chat, v_user,
'seed-' || g, now() - (g || ' milliseconds')::interval
FROM generate_series(1, %d) AS g;
END $$;
""" % need
r = subprocess.run(
[
"docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"postgres:18-alpine",
"psql", dsn_in_container, "-c", seed_sql,
],
capture_output=True, text=True, timeout=120,
)
seed_out = r.stdout + r.stderr
check("ERROR" not in seed_out, "сидирование не удалось:\n" + seed_out)
# ANALYZE обязателен: без него планировщик не знает реальных размеров.
_psql(dsn_in_container, "ANALYZE messages;")
_psql(dsn_in_container, "ANALYZE chat_members;")
_psql(dsn_in_container, "ANALYZE chats;")
print(" сидирование завершено")
# EXPLAIN по САМОМУ БОЛЬШОМУ чату, чтобы планировщик имел основания
# выбрать индекс, а не Seq Scan.
biggest = _psql(
dsn_in_container,
"SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;",
).strip().splitlines()[-1]
check(biggest and len(biggest) >= 32, "не удалось выбрать самый большой чат: %r" % biggest)
sql = (
"EXPLAIN (ANALYZE, BUFFERS) "
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
"WHERE chat_id = '%s' "
"ORDER BY created_at, id LIMIT 51;" % biggest
)
r = subprocess.run(
[
"docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"postgres:18-alpine",
"psql", dsn_in_container, "-c", sql,
],
capture_output=True, text=True, timeout=120,
)
out = r.stdout + r.stderr
print("EXPLAIN (фрагмент):")
for line in out.splitlines()[:12]:
print(" ", line)
check("Index" in out or "Bitmap Index" in out or "idx_messages_chat_created" in out,
"EXPLAIN не показал использования индекса:\n" + out)
# ---------- 3. Rollback составной операции create_chat ----------
def test_chat_rollback(c):
"""Проверяем, что при падении INSERT в chat_members ПОСЛЕ успешного
INSERT в chats не остаётся ни строки чата, ни частичного членства.
Как добиться ошибки именно на вставке участников, а не на предварительной
проверке unknown_member? Проверка `SELECT id FROM users WHERE id = ANY(...)`
в postgres.py отсекает неизвестных до INSERT. Значит, нужен сбой именно
в executemany.
Решение: запускаем патч-скрипт ВНУТРИ контейнера API через docker exec —
там уже установлен psycopg и доступен DATABASE_URL. Скрипт monkey-patch-ит
psycopg.Cursor.executemany так, чтобы он бросил исключение, и вызывает
storage.create_chat с двумя реально существующими пользователями.
Транзакция дойдёт до INSERT в chats, упадёт на executemany и должна
откатиться.
После этого через psql проверяем:
- в таблице chats нет строки с этим title;
- в таблице chat_members нет строк с этим chat_id;
- через API GET /api/v1/chats пользователь не видит этот чат.
"""
dsn = _read_dsn()
# Готовим двух валидных пользователей через API, чтобы FK был валиден.
suffix = uuid.uuid4().hex[:10]
alice_username = "rb_a_" + suffix
bob_username = "rb_b_" + suffix
alice_password = "T3st-" + uuid.uuid4().hex
bob_password = "T3st-" + uuid.uuid4().hex
alice = c.call("POST", "/api/v1/users", 201,
body={"username": alice_username, "display_name": "RB-A",
"password": alice_password},
schema="User")["json"]
bob = c.call("POST", "/api/v1/users", 201,
body={"username": bob_username, "display_name": "RB-B",
"password": bob_password},
schema="User")["json"]
# Уникальный title, чтобы точно найти строку, если rollback не сработает.
marker_title = "rollback-chat-" + uuid.uuid4().hex
# Скрипт запускается ВНУТРИ контейнера API: там есть psycopg и DATABASE_URL.
# Патчит executemany так, чтобы он упал после успешного INSERT в chats.
patch_script = r'''
import os, sys, psycopg
from uuid import UUID
_orig = psycopg.Cursor.executemany
def boom(self, *args, **kwargs):
raise RuntimeError("patched_executemany_failure")
psycopg.Cursor.executemany = boom
sys.path.insert(0, "/app/src")
from app.storage.postgres import storage
try:
storage.create_chat(os.environ["RB_TITLE"],
[UUID(os.environ["RB_ALICE"]), UUID(os.environ["RB_BOB"])])
except RuntimeError as exc:
if "patched_executemany_failure" in str(exc):
print("PATCHED_FAILURE_OK"); sys.exit(0)
raise
except Exception as exc:
print("UNEXPECTED:%s:%s" % (type(exc).__name__, exc)); sys.exit(2)
print("NO_FAILURE"); sys.exit(3)
'''
proc = subprocess.run(
["docker", "exec",
"-e", "RB_TITLE=" + marker_title,
"-e", "RB_ALICE=" + alice["id"],
"-e", "RB_BOB=" + bob["id"],
API_CONTAINER,
"python", "-c", patch_script],
capture_output=True, text=True, timeout=60,
)
out = (proc.stdout + proc.stderr).strip()
print(" rollback-процесс: exit=%d, out=%r" % (proc.returncode, out[:200]))
check("PATCHED_FAILURE_OK" in out,
"не удалось вызвать ошибку именно на executemany: %s" % out)
# Проверяем в БД: ни chats, ни chat_members с этим маркером не осталось.
chats_count = _scalar(
dsn,
"SELECT count(*) FROM chats WHERE title = '%s';" % marker_title,
)
check(int(chats_count) == 0,
"rollback не сработал: строка chats с title=%s осталась (%s)" % (marker_title, chats_count))
members_count = _scalar(
dsn,
"SELECT count(*) FROM chat_members cm "
"JOIN chats c ON c.id = cm.chat_id "
"WHERE c.title = '%s';" % marker_title,
)
check(int(members_count) == 0,
"rollback не сработал: остались строки chat_members (%s)" % members_count)
# Дополнительно: у пользователя нет чата с таким title через API.
cred = base64.b64encode((alice_username + ":" + alice_password).encode()).decode()
session = c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + cred},
schema="Session")["json"]
listed = c.call("GET", "/api/v1/chats?limit=100",
headers={"Authorization": "Bearer " + session["token"]},
schema="ChatPage")["json"]
check(all(item["title"] != marker_title for item in listed["items"]),
"чат с маркером rollback виден в списке чатов пользователя")
# ---------- 4. Недоступность Redis (с действующим токеном) ----------
def _compose(action, service="redis"):
return subprocess.run(
["docker", "compose", "--env-file", ".env", action, service],
capture_output=True, text=True, timeout=60,
)
def _wait_ready(c, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
return
except Failure:
time.sleep(1)
raise Failure("API не вернулся в ready за %d секунд после восстановления зависимости" % timeout)
def test_redis_unavailable(c):
"""Останавливаем Redis: /health/ready → 503, действующий токен не работает.
Важно: используем ЗАРАНЕЕ полученную рабочую сессию, а не заведомо
неправильный Bearer. Иначе 401 мог бы получиться и без обращения к Redis.
После восстановления Redis прежняя сессия снова должна работать.
"""
print(" (проверка недоступности Redis: docker compose stop redis на ~6 секунд)")
# Готовим рабочую сессию ДО остановки Redis.
suffix = uuid.uuid4().hex[:10]
username = "redis_" + suffix
password = "T3st-" + uuid.uuid4().hex
c.call("POST", "/api/v1/users", 201,
body={"username": username, "display_name": "Redis", "password": password},
schema="User")
cred = base64.b64encode((username + ":" + password).encode()).decode()
session = c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + cred},
schema="Session")["json"]
token = session["token"]
# Контроль: до остановки Redis токен рабочий.
me = c.call("GET", "/api/v1/users/me", 200,
headers={"Authorization": "Bearer " + token},
schema="User")["json"]
check(me["username"] == username, "токен должен принадлежать созданному пользователю")
stopped = _compose("stop", "redis")
check(stopped.returncode == 0,
"docker compose stop redis не сработал:\n" + (stopped.stdout + stopped.stderr))
try:
time.sleep(3)
# /health/ready должен стать 503.
ready = c.call("GET", "/health/ready", 503)
check(ready["status"] == 503, "readiness must be 503 without Redis")
# Действующий токен при мёртвом Redis не должен давать доступ:
# сервер не может проверить сессию и обязан ответить 503.
r = c.call("GET", "/api/v1/users/me", (401, 503),
headers={"Authorization": "Bearer " + token})
check(r["status"] == 503,
"запрос с действующим токеном при мёртвом Redis должен быть 503, получено %s" % r["status"])
finally:
# Только восстановление. Никаких return/break/continue внутри finally.
started = _compose("start", "redis")
check(started.returncode == 0,
"docker compose start redis не сработал:\n" + (started.stdout + started.stderr))
# Ждём готовности — УЖЕ ВНЕ finally.
_wait_ready(c, timeout=60)
# Прежний токен снова должен работать: сессия пережила рестарт Redis (AOF).
me2 = c.call("GET", "/api/v1/users/me", 200,
headers={"Authorization": "Bearer " + token},
schema="User")["json"]
check(me2["id"] == me["id"], "прежняя сессия должна снова работать после восстановления Redis")
# ---------- 5. Недоступность PostgreSQL ----------
def test_postgres_unavailable(c):
"""Останавливаем PostgreSQL: /health/ready → 503, действующий токен → 503.
Проверяем:
- /health/ready возвращает 503;
- запрос с действующим токеном возвращает 503 в пределах таймаута;
- анонимный запрос не получает доступ;
- после запуска БД прежний токен работает, ранее записанные данные сохранены.
Восстановление БД выполняется в finally, чтобы не оставлять стенд остановленным.
"""
print(" (проверка недоступности PostgreSQL: docker compose stop postgres на ~6 секунд)")
# Готовим рабочую сессию и сообщение ДО остановки БД.
alice, bob, eve, chat = c.bootstrap()
marker = "pgdown-" + uuid.uuid4().hex
sent = c.message(alice, chat, marker)["json"]
# Контроль: до остановки всё работает.
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
stopped = _compose("stop", "postgres")
check(stopped.returncode == 0,
"docker compose stop postgres не сработал:\n" + (stopped.stdout + stopped.stderr))
try:
time.sleep(3)
# /health/ready должен стать 503.
ready = c.call("GET", "/health/ready", 503)
check(ready["status"] == 503, "readiness must be 503 without PostgreSQL")
# Запрос с действующим токеном: 503 в пределах таймаута.
t0 = time.monotonic()
r = c.call("GET", "/api/v1/users/me", 503, actor=alice)
elapsed = time.monotonic() - t0
check(r["status"] == 503,
"запрос с действующим токеном при мёртвой БД должен быть 503")
# statement_timeout=5000 + пул timeout=5.0: укладываемся в разумный предел.
check(elapsed < 30, "ответ при мёртвой БД занял %.1f c, ожидался в пределах таймаута" % elapsed)
# Анонимный запрос не получает доступ.
anon = c.call("GET", "/api/v1/users/me", (401, 503))
check(anon["status"] in (401, 503),
"анонимный запрос при мёртвой БД должен быть 401 или 503, получено %s" % anon["status"])
finally:
started = _compose("start", "postgres")
check(started.returncode == 0,
"docker compose start postgres не сработал:\n" + (started.stdout + started.stderr))
# Ждём готовности — УЖЕ ВНЕ finally.
_wait_ready(c, timeout=60)
# Прежний токен работает, ранее записанное сообщение сохранено.
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
messages = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=bob, schema="MessagePage")["json"]["items"]
check(any(m["id"] == sent["id"] and m["text"] == marker for m in messages),
"ранее записанное сообщение не найдено после восстановления PostgreSQL")
# ---------- 6. Expiry TTL ----------
def test_short_ttl_expiry(c):
"""TTL=5. Проверяем полный цикл: токен валиден -> истекает -> 401.
Идёт ПОСЛЕДНИМ: при коротком TTL любая долгая подготовка съедает
окно жизни сессии, поэтому соседние тесты работают раньше.
"""
suffix = uuid.uuid4().hex[:10]
username = "ttl_" + suffix
password = "T3st-" + uuid.uuid4().hex
user = c.call("POST", "/api/v1/users", 201,
body={"username": username, "display_name": "TTL", "password": password},
schema="User")["json"]
cred = base64.b64encode((username + ":" + password).encode()).decode()
session = c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + cred},
schema="Session")["json"]
token = session["token"]
# Контроль: сервер выдал короткий TTL, иначе тест бессмыслен.
expires_at = dt.datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00"))
ttl_actual = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds()
check(3.0 <= ttl_actual <= 10,
"SESSION_TTL_SECONDS не короткий (получено %.1f c). "
"Поставьте, пожалуйста, SESSION_TTL_SECONDS=5 в .env и перезапустите API (команда: docker compose --env-file .env up -d api)." % ttl_actual)
# До истечения токен должен работать.
me = c.call("GET", "/api/v1/users/me", 200,
headers={"Authorization": "Bearer " + token},
schema="User")["json"]
check(me["id"] == user["id"], "token must identify its owner before expiry")
# Ждём гарантированного истечения TTL + запас.
remaining = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds()
time.sleep(max(remaining, 0.0) + 1.0)
# Истёкший токен — 401.
c.call("GET", "/api/v1/users/me", 401,
headers={"Authorization": "Bearer " + token})
# Второй вход тем же пользователем даёт рабочий токен: истёк именно
# первый токен, а не пользователь/хранилище целиком.
session2 = c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + cred},
schema="Session")["json"]
c.call("GET", "/api/v1/users/me", 200,
headers={"Authorization": "Bearer " + session2["token"]},
schema="User")
# ---------- main ----------
def main():
args = arguments("Lab2 level 4 checks", configure=_configure)
c = Client(args.base_url, args.ca_file)
# Порядок и режимы:
# --only-expiry — ТОЛЬКО expiry-тест (запускать при SESSION_TTL_SECONDS=5)
# --skip-expiry — все тяжёлые тесты, без expiry (при SESSION_TTL_SECONDS=3600)
# без флагов — сначала тяжёлые, expiry последним (при SESSION_TTL_SECONDS=5
# это сработает только если тяжёлые тесты успевают до истечения TTL,
# поэтому на практике используйте --only-expiry/--skip-expiry)
run_expiry = not args.skip_expiry
run_others = not args.only_expiry
if run_others:
print("=== test_concurrent_registration ===")
test_concurrent_registration(c)
print("PASS: concurrent registration produces exactly one user")
print("=== test_chat_rollback ===")
test_chat_rollback(c)
print("PASS: create_chat rollback leaves no chat and no partial membership")
print("=== test_explain ===")
test_explain(c)
print("PASS: EXPLAIN uses idx_messages_chat_created")
print("=== test_redis_unavailable ===")
test_redis_unavailable(c)
print("PASS: readiness 503 без Redis, действующий токен отвергнут, затем восстановление")
print("=== test_postgres_unavailable ===")
test_postgres_unavailable(c)
print("PASS: readiness 503 без PostgreSQL, данные и сессия сохранены")
if run_expiry:
print("=== test_short_ttl_expiry ===")
test_short_ttl_expiry(c)
print("PASS: session valid before TTL, expires after TTL, 401 on expired token")
print("PASS: all requested lab2 level-4 checks passed")
def _configure(p):
p.add_argument("--only-expiry", action="store_true",
help="запустить только expiry-тест (SESSION_TTL_SECONDS=5)")
p.add_argument("--skip-expiry", action="store_true",
help="запустить всё, кроме expiry (SESSION_TTL_SECONDS=3600)")
if __name__ == "__main__":
run(main)
+465
View File
@@ -0,0 +1,465 @@
#!/usr/bin/env python3
"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO.
Шаги:
1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql).
2. Сравниваем 3 варианта индексов на большом и маленьком чате:
- baseline (нет индекса) -> Seq Scan
- (chat_id) -> Bitmap/Index + Sort
- (chat_id, created_at, id) -> Index Scan без Sort (текущий)
Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read.
3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки,
pg_indexes_size, pg_total_relation_size.
4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine,
сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений.
5. Границы потери данных: во время pg_dump через API пишем новые сообщения,
после restore проверяем, что их нет.
Все psql-вызовы идут через docker run postgres:18-alpine в compose-сети проекта
(--network mtuci2026lab2_default), DSN указывает на хост postgres.
"""
import json
import os
import random
import subprocess
import sys
import time
import uuid
from pathlib import Path
import base64
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
EVIDENCE.mkdir(exist_ok=True)
SEED_SQL = ROOT / "student-tests" / "track2_seed.sql"
# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name).
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default")
# ---------- helpers ----------
def _read_dsn() -> str:
dsn = os.environ.get("DATABASE_URL")
if not dsn:
env_file = ROOT / ".env"
if env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line.startswith("DATABASE_URL="):
dsn = line.split("=", 1)[1].strip()
break
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
# Postgres доступен только внутри compose-сети; одноразовые psql/pg_dump
# контейнеры запускаются в этой же сети, имя хоста postgres резолвится как есть.
return dsn
def _psql(dsn: str, sql: str, extra=None):
cmd = [
"docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"postgres:18-alpine",
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1",
]
if extra:
cmd += extra
cmd += ["-c", sql]
return subprocess.run(cmd, capture_output=True, text=True, timeout=300)
def _psql_file(dsn: str, path: Path):
cmd = [
"docker", "run", "--rm", "-i",
"--network", COMPOSE_NETWORK,
"-v", f"{path}:/seed.sql:ro",
"postgres:18-alpine",
"psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql",
]
return subprocess.run(cmd, capture_output=True, text=True, timeout=600)
def _scalar(dsn: str, sql: str):
r = _psql(dsn, sql)
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
return r.stdout.strip().splitlines()[-1]
# ---------- 1. seed ----------
def step_seed(dsn: str):
print("=== 1. seed ===")
before = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
print(" messages before:", before)
r = _psql_file(dsn, SEED_SQL)
check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr)
after = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
print(" messages after:", after)
check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after)
dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;")
print(" распределение по чатам:\n" + dist.stdout)
return after
# ---------- 2. explain ----------
def _big_and_small(dsn):
big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;")
small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;")
return big, small
def _chat_size(dsn: str, chat_id: str) -> int:
return int(_scalar(dsn, "SELECT count(*) FROM messages WHERE chat_id = '%s';" % chat_id))
def _explain(dsn: str, chat_id: str, label: str):
# Берём средний created_at, чтобы keyset реально резал историю
mid = _scalar(dsn,
"SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id))
sql = (
"EXPLAIN (ANALYZE, BUFFERS) "
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
"WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') "
"ORDER BY created_at, id LIMIT 51;" % (chat_id, mid)
)
r = _psql(dsn, sql)
check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr)
out = r.stdout
print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8]))
for line in out.splitlines():
print(" ", line)
return out
def _drop_indexes(dsn):
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;")
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
def _format_section(title: str, chat_size: int, plan: str, note: str = "") -> str:
"""Собирает один раздел отчёта в формате образца."""
head = "## %s\n\n" % title
if note:
head += note.rstrip() + "\n\n"
return head + "```\n" + plan.rstrip() + "\n```\n"
def step_explain(dsn: str, total_messages: int):
print("=== 2. EXPLAIN 3 варианта ===")
big, small = _big_and_small(dsn)
big_size = _chat_size(dsn, big)
small_size = _chat_size(dsn, small)
# (1) без индекса
_drop_indexes(dsn)
_psql(dsn, "ANALYZE messages;")
no_index_big = _explain(dsn, big, "no_index/big")
no_index_small = _explain(dsn, small, "no_index/small")
# (2) только (chat_id)
_psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);")
_psql(dsn, "ANALYZE messages;")
chat_only_big = _explain(dsn, big, "chat_only/big")
chat_only_small = _explain(dsn, small, "chat_only/small")
# (3) полный рабочий
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
_psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);")
_psql(dsn, "ANALYZE messages;")
full_big = _explain(dsn, big, "full/big")
full_small = _explain(dsn, small, "full/small")
# Заголовок отчёта — как в образце
parts = []
parts.append(
"# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса\n\n"
"Данные: %d сообщений, неравномерные чаты. "
"Запрос: `SELECT ... FROM messages WHERE chat_id = <id> AND (created_at, id) > (<середина>, …) "
"ORDER BY created_at, id LIMIT 51`.\n"
"Большой чат ~%d, маленький чат ~%d.\n" % (total_messages, big_size, small_size)
)
parts.append(_format_section(
"1. Без индекса, большой чат", big_size, no_index_big,
"`Parallel Seq Scan` по всей таблице, затем `Sort`."))
parts.append(_format_section(
"2. Без индекса, маленький чат", small_size, no_index_small,
"Тот же `Parallel Seq Scan` по всей таблице."))
parts.append(_format_section(
"3. Индекс (chat_id), большой чат", big_size, chat_only_big,
"`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`."))
parts.append(_format_section(
"4. Индекс (chat_id), маленький чат", small_size, chat_only_small,
"`Index Scan using idx_messages_chat_only` + `Sort`."))
parts.append(_format_section(
"5. Полный индекс (chat_id, created_at, id), большой чат", big_size, full_big,
"`Index Scan using idx_messages_chat_created`, **без `Sort`**."))
parts.append(_format_section(
"6. Полный индекс (chat_id, created_at, id), маленький чат", small_size, full_small,
"`Index Scan` без `Sort`."))
(EVIDENCE / "lab2_track2_explain.md").write_text(
"\n".join(parts), encoding="utf-8")
return {"big_chat": big, "small_chat": small}
# ---------- 3. write cost ----------
def _index_sizes(dsn: str):
out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));")
idx, total = out.split("|")
return idx.strip(), total.strip()
def step_write_cost(dsn: str):
print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===")
seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;")
seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;")
results = []
for name, setup in (
("no_index", lambda: _drop_indexes(dsn)),
("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"),
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))),
("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"),
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))),
):
setup()
# мерить именно вставку 10k
t0 = time.monotonic()
r = _psql(dsn,
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() "
"FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name))
dt = time.monotonic() - t0
check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr)
idx, total = _index_sizes(dsn)
results.append({"variant": name, "insert_10000_s": round(dt, 3),
"indexes_size": idx, "total_size": total})
print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total))
(EVIDENCE / "lab2_track2_write_cost.json").write_text(
json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return results
# ---------- 4. backup/restore ----------
def step_backup_restore(dsn: str):
print("=== 4. backup -> restore -> сверка ===")
out_file = EVIDENCE / "lab2_track2_dump.dump"
if out_file.exists():
out_file.unlink()
# pg_dump через docker run (использует образ PG 18), в compose-сети
t0 = time.monotonic()
r = subprocess.run(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/out",
"postgres:18-alpine",
"pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn],
capture_output=True, text=True, timeout=600,
)
dump_time = time.monotonic() - t0
check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr)
size = out_file.stat().st_size
print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time))
# новый контейнер как отдельное хранилище — в той же compose-сети,
# порт наружу не публикуем
restore_container = "mtuci-pg-restore"
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
subprocess.run(
["docker", "run", "-d", "--name", restore_container,
"--network", COMPOSE_NETWORK,
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
"-e", "POSTGRES_DB=high_load_messenger",
"postgres:18-alpine"],
capture_output=True, text=True,
)
# ждём готовности
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
capture_output=True, text=True)
if c.returncode == 0:
break
time.sleep(1)
else:
raise Failure("restore container не поднялся")
# pg_restore -c (clean) на пустую БД; restore-контейнер доступен по имени
# в compose-сети
t0 = time.monotonic()
r = subprocess.run(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/in",
"postgres:18-alpine",
"pg_restore", "-d", "postgresql://postgres:restore@mtuci-pg-restore:5432/high_load_messenger",
"--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"],
capture_output=True, text=True, timeout=600,
)
restore_time = time.monotonic() - t0
print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time))
check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr)
# Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера
def q_restore(sql):
c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
"-d", "high_load_messenger", "-tA", "-c", sql],
capture_output=True, text=True, timeout=120)
check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr)
return c.stdout.strip().splitlines()[-1]
orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
rest_total = int(q_restore("SELECT count(*) FROM messages;"))
check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total))
orig_dist = _scalar(dsn,
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
rest_dist = q_restore(
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
check(orig_dist == rest_dist, "распределение по чатам не совпадает")
# 100 случайных сообщений
ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;")
ids = ids_raw.split(",")
check(len(ids) == 100, "не удалось выбрать 100 id")
ids_list = ",".join("'%s'" % i for i in ids)
orig_rows = _scalar(dsn,
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
rest_rows = q_restore(
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
check(orig_rows == rest_rows, "100 случайных сообщений различаются")
print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают")
print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024))
(EVIDENCE / "lab2_track2_restore.txt").write_text(
"Track 2 restore\n"
"dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n"
"count_original=%d\ncount_restored=%d\n"
"dist_match=true\nrandom100_match=true\n"
% (dump_time, size, restore_time, orig_total, rest_total),
encoding="utf-8")
# чистим restore-контейнер (volume анонимный)
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3),
"dump_bytes": size, "count": orig_total}
# ---------- 5. RPO ----------
def step_rpo(dsn: str, base_url: str, ca_file):
print("=== 5. RPO: запись во время pg_dump ===")
c = Client(base_url, ca_file)
# Данные и сессии готовим заранее, до дампа.
alice, bob, eve, chat = c.bootstrap()
# Контроль: сессия должна быть валидна, иначе 401 — это не RPO.
me = c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
check(me["status"] == 200, "сессия должна быть валидна перед RPO-тестом")
marker = "rpo-" + uuid.uuid4().hex
# Запускаем pg_dump
out_file = EVIDENCE / "lab2_track2_rpo.dump"
if out_file.exists():
out_file.unlink()
# pg_dump: тот же приём, что в шаге 4, в compose-сети.
proc = subprocess.Popen(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/out",
"postgres:18-alpine",
"pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
time.sleep(1.0) # снапшот уже снят
# Свежий логин прямо перед записью: не зависим от TTL старой сессии.
cred = base64.b64encode(
(alice["registration"]["username"] + ":" +
alice["registration"]["password"]).encode()
).decode()
fresh = c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + cred},
schema="Session")["json"]
alice_for_write = {"id": alice["id"], "user": alice["user"], "token": fresh["token"]}
written = c.message(alice_for_write, chat, marker)["json"]
code = proc.wait(timeout=300)
check(code == 0, "pg_dump для RPO упал")
# Временный restore-контейнер: в compose-сети, без публикации портов.
restore_container = "mtuci-pg-rpo"
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
subprocess.run(
["docker", "run", "-d", "--name", restore_container,
"--network", COMPOSE_NETWORK,
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
"-e", "POSTGRES_DB=high_load_messenger",
"postgres:18-alpine"],
capture_output=True, text=True,
)
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
capture_output=True, text=True)
if c2.returncode == 0:
break
time.sleep(1)
else:
raise Failure("restore container для RPO не поднялся")
# pg_restore идёт в restore-контейнер по имени в compose-сети.
r = subprocess.run(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/in",
"postgres:18-alpine",
"pg_restore", "-d",
"postgresql://postgres:restore@mtuci-pg-rpo:5432/high_load_messenger",
"--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"],
capture_output=True, text=True, timeout=600,
)
check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr)
c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
"-d", "high_load_messenger", "-tA", "-c",
"SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]],
capture_output=True, text=True)
present = int(c3.stdout.strip().splitlines()[-1])
check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)")
print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8])
(EVIDENCE / "lab2_track2_rpo.txt").write_text(
"Track 2 RPO\n"
"marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n"
% (written["id"], present),
encoding="utf-8")
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
# ---------- main ----------
def main():
args = arguments("Track 2: plan, write cost, backup/restore, RPO")
dsn = _read_dsn()
total = step_seed(dsn)
step_explain(dsn, total)
step_write_cost(dsn)
step_backup_restore(dsn)
step_rpo(dsn, args.base_url, args.ca_file)
print("PASS: track 2 (explain + write cost + backup/restore + RPO)")
if __name__ == "__main__":
run(main)
+10
View File
@@ -0,0 +1,10 @@
-- Track 2: сравнение планов и стоимости записи.
-- Все три варианта прогоняются на одном большом чате (60k) и одном маленьком (~1400).
-- 0. Определяем два чата
\set ON_ERROR_STOP on
SELECT id AS big_chat_id
FROM messages GROUP BY id_chat GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;
SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;
+81
View File
@@ -0,0 +1,81 @@
-- Track 2: сидирование >=100 000 сообщений с неравномерными чатами.
-- Запускать через: docker run --rm ... psql "$DATABASE_URL" -f track2_seed.sql
-- Идемпотентно по количеству: если сообщений уже >= 100 000, ничего не делает.
DO $$
DECLARE
v_user UUID;
v_total INT;
v_have INT;
v_need INT;
v_chat UUID;
rec RECORD;
BEGIN
SELECT count(*) INTO v_have FROM messages;
v_total := 100000;
SELECT count(*) INTO v_have FROM messages;
IF v_have < 100000 THEN
-- меньше 100к: догоняем до 100к
v_need := 100000 - v_have; -- сколько добавлять за прогон
ELSE
v_need := 50000; -- добавляем фиксированные 50к за прогон
END IF;
-- Гарантируем хотя бы одного пользователя
SELECT id INTO v_user FROM users LIMIT 1;
IF v_user IS NULL THEN
v_user := gen_random_uuid();
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed',
'scrypt$16384$8$1$00$00', now());
END IF;
-- Создаём 10 чатов, если их меньше (нужны только для распределения)
-- Неравномерность: 60k / 20k / 10k / по ~1400 на остальные 7.
-- Считаем веса, доливаем только v_need строк.
CREATE TEMP TABLE _plan(chat_no INT, weight INT) ON COMMIT DROP;
INSERT INTO _plan VALUES (1, 60), (2, 20), (3, 10),
(4, 2), (5, 2), (6, 1), (7, 1),
(8, 1), (9, 1), (10, 1);
FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP
-- Находим/создаём chat_id для этого номера: используем title 'seed-chat-<no>'
SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1;
IF v_chat IS NULL THEN
v_chat := gen_random_uuid();
INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'seed-chat-' || rec.chat_no, now());
END IF;
IF NOT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = v_chat AND user_id = v_user) THEN
INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user);
END IF;
END LOOP;
-- Распределяем v_need по весам (целочисленно), последний чат добирает остаток.
DECLARE
v_sum INT;
v_acc INT := 0;
v_alloc INT;
BEGIN
SELECT sum(weight) INTO v_sum FROM _plan;
FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP
IF rec.chat_no = 10 THEN
v_alloc := v_need - v_acc; -- остаток
ELSE
v_alloc := (v_need::numeric * rec.weight / v_sum)::int;
v_acc := v_acc + v_alloc;
END IF;
SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), v_chat, v_user,
'seed-' || rec.chat_no || '-' || g,
now() - (g || ' milliseconds')::interval
FROM generate_series(1, v_alloc) AS g;
END LOOP;
END;
END $$;
ANALYZE messages;
ANALYZE chat_members;
ANALYZE chats;
+223
View File
@@ -0,0 +1,223 @@
#!/usr/bin/env python3
"""Уровень 4 lab3: два X-Instance-Id, идемпотентность под нагрузкой, отказ одной реплики.
Запуск:
python student-tests/lab3_test_level4.py --base-url http://localhost:8080
Требуется рабочий стенд из compose.yaml (LB + 2 реплики + Postgres + Redis).
"""
import base64
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
# ---------- 1. Два X-Instance-Id через LB ----------
def test_two_instances(c):
"""Один пользователь, серия запросов — видим >=2 разных X-Instance-Id.
Nginx с keepalive к upstream прибивает клиента к одной реплике на всё
время соединения. Поэтому каждый проб-запрос шлём с Connection: close,
чтобы LB открывал новое upstream-соединение и честно распределял.
"""
alice, bob, eve, chat = c.bootstrap()
seen = set()
for _ in range(60):
r = c.call("GET", "/api/v1/users/me", actor=alice, schema="User",
headers={"Connection": "close"})
seen.add(r["headers"]["X-Instance-Id"])
if len(seen) >= 2:
break
check(len(seen) >= 2, "LB не маршрутизирует на разные API-реплики: %s" % seen)
print(" observed instances:", sorted(seen))
# ---------- 2. Идемпотентность: один ключ на двух репликах ----------
def test_idempotency_across_replicas(c):
"""Одновременный POST с одним Idempotency-Key через LB создаёт ровно одно сообщение.
Для этого параллельно шлём 10 запросов с одним ключом из 10 потоков.
LB распределит их по разным репликам; благодаря общей таблице
idempotency_keys и UNIQUE-индексу, INSERT выигрывает ровно один.
"""
alice, bob, eve, chat = c.bootstrap()
key = "idem-" + uuid.uuid4().hex
path = "/api/v1/chats/" + chat["id"] + "/messages"
body = {"text": "idem-concurrent"}
lock = threading.Lock()
results = []
def one(_):
local = Client(c.base, timeout=10)
try:
r = local.call(
"POST", path, (201, 409),
actor=alice, body=body,
headers={"Idempotency-Key": key},
schema="Message",
)
with lock:
results.append((r["status"], r["headers"].get("X-Instance-Id")))
except Failure as exc:
with lock:
results.append(("failure:%s" % exc, None))
with ThreadPoolExecutor(max_workers=10) as pool:
list(pool.map(one, range(10)))
ok = [r for r in results if r[0] == 201]
conflicts = [r for r in results if r[0] == 409]
bad = [r for r in results if r[0] not in (201, 409)]
check(not bad, "unexpected idempotency results: %s" % results)
check(len(ok) >= 1, "no 201 at all")
check(len(ok) + len(conflicts) == 10, "wrong count")
# Проверяем в БД: ровно одно сообщение с этим текстом в этом чате.
# Через API: читаем всю историю и считаем.
from urllib.parse import quote
items = []
cursor = None
for _ in range(50):
q = path + "?limit=100"
if cursor:
q += "&cursor=" + quote(cursor, safe="")
page = c.call("GET", q, actor=bob, schema="MessagePage")["json"]
items.extend(m for m in page["items"] if m["text"] == "idem-concurrent")
cursor = page["next_cursor"]
if cursor is None:
break
check(len(items) == 1, "idempotency created %d messages, expected 1" % len(items))
# X-Instance-Id: запросы шли на разные реплики (проверяем, что 201 пришёл
# хотя бы с одной, а 409 — с другой; это ожидание не обязательное, но
# полезный сигнал, что LB реально распределяет).
ok_instances = {r[1] for r in ok}
conflict_instances = {r[1] for r in conflicts}
print(" 201 instances:", ok_instances, " 409 instances:", conflict_instances)
print(" PASS: idempotency OK, ровно одно сообщение создано")
# ---------- 3. Отказ одной реплики ----------
def test_stop_one_replica(c):
"""Останавливаем одну реплику; вторая продолжает обслуживать запросы.
Восстановление делаем явно через `docker start` в finally, а ожидание
готовности — ПОСЛЕ try/finally, чтобы не использовать return в finally.
"""
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
# Список контейнеров сервиса api (compose project name неизвестен,
# ищем по image/label).
ps = subprocess.run(
["docker", "ps", "--filter", "label=com.docker.compose.service=api",
"--format", "{{.ID}}"],
capture_output=True, text=True, timeout=10,
)
ids = [x.strip() for x in ps.stdout.splitlines() if x.strip()]
check(len(ids) >= 2, "нужно >=2 контейнера api, найдено %d" % len(ids))
target = ids[0]
print(" останавливаем replica container:", target[:12])
# Готовим одно подтверждённое сообщение до отказа.
key0 = "before-" + uuid.uuid4().hex
first = c.call("POST", path, 201, actor=alice, body={"text": "before-fail"},
headers={"Idempotency-Key": key0}, schema="Message")["json"]
# Останавливаем реплику.
stop = subprocess.run(["docker", "stop", "-t", "5", target],
capture_output=True, text=True, timeout=30)
check(stop.returncode == 0, "docker stop failed: %s" % stop.stderr)
try:
# Даём LB обнаружить отказ (proxy_next_upstream, max_fails=2).
time.sleep(3)
# Читаем и пишем — должны попадать во вторую реплику.
deadline = time.monotonic() + 10
recovered = False
seen_instances = set()
while time.monotonic() < deadline:
try:
r = c.call("POST", path, 201, actor=alice,
body={"text": "after-fail"},
headers={"Idempotency-Key": "after-" + uuid.uuid4().hex},
schema="Message")
seen_instances.add(r["headers"].get("X-Instance-Id"))
recovered = True
break
except Failure:
time.sleep(0.5)
check(recovered, "не удалось записать после отказа за 10 секунд")
# Убеждаемся, что подтверждённое до отказа сообщение доступно.
from urllib.parse import quote
items, cursor = [], None
while True:
q = path + "?limit=100"
if cursor:
q += "&cursor=" + quote(cursor, safe="")
page = c.call("GET", q, actor=bob, schema="MessagePage")["json"]
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
check(any(m["id"] == first["id"] for m in items),
"подтверждённое сообщение потеряно после отказа")
print(" after-fail instances:", seen_instances)
print(" PASS: стенд пережил отказ одной реплики")
finally:
# Только восстановление реплики. Никаких return/break/continue.
subprocess.run(["docker", "start", target],
capture_output=True, text=True, timeout=30)
# Ждём готовности через LB — УЖЕ ВНЕ finally.
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
# И двух реплик снова должно быть >=2.
seen = set()
for _ in range(30):
seen.add(c.call("GET", "/api/v1/users/me", actor=alice,
schema="User")["headers"]["X-Instance-Id"])
if len(seen) >= 2:
break
if len(seen) >= 2:
return
except Failure:
pass
time.sleep(1)
raise Failure("стенд не восстановился до двух реплик за 60 секунд")
# ---------- main ----------
def main():
args = arguments("Lab3 level 4 checks")
c = Client(args.base_url, args.ca_file)
print("=== test_two_instances ===")
test_two_instances(c)
print("PASS: >=2 X-Instance-Id через LB")
print("=== test_idempotency_across_replicas ===")
test_idempotency_across_replicas(c)
print("PASS: идемпотентность на общих ключах")
print("=== test_stop_one_replica ===")
test_stop_one_replica(c)
print("PASS: отказ одной API-реплики обработан")
print("PASS: all lab3 level-4 checks passed")
if __name__ == "__main__":
run(main)
+329
View File
@@ -0,0 +1,329 @@
#!/usr/bin/env python3
"""Level 4 для lab4: негативные сценарии, компенсация при отказе БД, параллельные загрузки.
Запуск:
$env:API_CONTAINER = (docker ps --filter "label=com.docker.compose.service=api" -q | Select-Object -First 1)
python student-tests/lab4_test_level4.py --base-url http://localhost:8080
Сценарий компенсации (уровень 4):
Останавливаем PostgreSQL → делаем upload → S3 PUT успешен, INSERT в БД
падает → API удаляет объект из S3 (компенсация) → 503.
После восстановления PG считаем объекты в bucket через mc — количество
не должно вырасти. Это проверяет требование методички: «отказ между
записью S3/БД не оставляет вечных неучтённых объектов».
"""
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
API_CONTAINER = os.environ.get("API_CONTAINER", "")
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab4_default")
MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920")
def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str):
# filename URL-encoded: иначе urllib откажется формировать URL с NUL/CR/LF.
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
req = urllib.request.Request(
url, data=data, method="POST",
headers={
"Authorization": "Bearer " + actor["token"],
"Content-Type": ctype,
},
)
try:
with c.opener.open(req, timeout=30) as resp:
import json as _json
return resp.code, _json.loads(resp.read()), resp.headers
except urllib.error.HTTPError as e:
import json as _json
body = e.read()
try:
return e.code, _json.loads(body), e.headers
except Exception:
return e.code, None, e.headers
def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"):
return _raw_upload(c, actor, chat["id"], filename, data, ctype)
# ---------- негативные сценарии ----------
def test_oversize(c):
alice, bob, eve, chat = c.bootstrap()
big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10_485_761)
status, body, _h = _upload(c, alice, chat, big)
check(status == 413, "oversize upload must be 413, got %s" % status)
def test_corrupted_image(c):
alice, bob, eve, chat = c.bootstrap()
bad = b"\x89PNG\r\n\x1a\n" + b"not-a-real-png" * 100
status, body, _h = _upload(c, alice, chat, bad)
check(status == 415, "corrupted PNG must be 415, got %s" % status)
def test_mime_spoof(c):
alice, bob, eve, chat = c.bootstrap()
status, body, _h = _upload(c, alice, chat, b"hello, not an image" * 10, ctype="image/png")
check(status == 415, "spoofed MIME must be 415, got %s" % status)
def test_bad_filename(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
for bad in ("../evil.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", " "):
status, body, _h = _upload(c, alice, chat, data, filename=bad)
check(status == 400, "bad filename %r must be 400, got %s" % (bad, status))
# ---------- изоляция доступа ----------
def test_foreign_attachment_and_cross_chat(c):
alice, bob, eve, chat = c.bootstrap()
chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice,
body={"title": "Второй", "member_ids": [alice["id"], eve["id"]]},
schema="Chat")["json"]
data = SAMPLE.read_bytes()
status, att, _h = _upload(c, alice, chat2, data)
check(status == 201, "upload to chat2 must be 201, got %s" % status)
att_id = att["id"]
c.call("GET", "/api/v1/attachments/" + att_id, 404, actor=bob)
c.call("GET", "/api/v1/attachments/" + att_id + "/content", 404, actor=bob)
c.call(
"POST", "/api/v1/chats/" + chat["id"] + "/messages",
404, actor=bob,
body={"text": "", "attachment_ids": [att_id]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
def test_attachment_belongs_to_chat(c):
alice, bob, eve, chat = c.bootstrap()
other = c.call("POST", "/api/v1/chats", 201, actor=alice,
body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]},
schema="Chat")["json"]
data = SAMPLE.read_bytes()
status, att, _h = _upload(c, alice, chat, data)
check(status == 201, "upload to chat must be 201")
c.call(
"POST", "/api/v1/chats/" + other["id"] + "/messages",
404, actor=alice,
body={"text": "", "attachment_ids": [att["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
# ---------- компенсация: S3 PUT успешен, INSERT в БД падает ----------
def _compose(action, service):
return subprocess.run(
["docker", "compose", "--env-file", ".env", action, service],
capture_output=True, text=True, timeout=90,
)
def _wait_ready(c, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
return
except Failure:
time.sleep(1)
raise Failure("API не вернулся в ready за %d секунд" % timeout)
def _count_bucket_objects() -> int:
env = os.environ.copy()
env_file = ROOT / ".env"
if env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
env.setdefault(k.strip(), v.strip())
user = env.get("MINIO_ROOT_USER", "minioadmin")
password = env.get("MINIO_ROOT_PASSWORD", "minioadmin")
bucket = env.get("S3_BUCKET", "messenger-attachments")
mc_host = "http://%s:%s@minio:9000" % (user, password)
# Entrypoint образа mc обычно сам mc; тогда подкоманду зовём без префикса "mc".
ins = subprocess.run(
["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"],
capture_output=True, text=True, timeout=10,
)
ep = (ins.stdout or "").strip()
prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"]
r = subprocess.run(
["docker", "run", "--rm", "--network", COMPOSE_NETWORK,
"-e", "MC_HOST_local=" + mc_host, MC_IMAGE]
+ prefix
+ ["ls", "--recursive", "local/" + bucket],
capture_output=True, text=True, timeout=60,
)
if r.returncode != 0:
# Диагностика при первом запуске — увидим, что именно не так.
print(" mc stderr:", (r.stderr or "").strip()[:300])
return -1
return sum(1 for line in r.stdout.splitlines() if line.startswith("["))
def test_compensation_on_db_failure(c):
"""S3 PUT прошёл, INSERT в БД упал → объект удаляется из S3, ответ 503.
Сценарий: останавливаем PostgreSQL. API-реплика сначала делает PUT в S3
(успешно), затем INSERT в attachments (падает: connection error).
В except-ветке upload_attachment вызывается s3.delete_object —
orphan не остаётся. Ответ — 503 unavailable.
После восстановления PG сверяем количество объектов в bucket.
"""
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
before = _count_bucket_objects()
check(before >= 0, "не удалось посчитать объекты в bucket (mc)")
stopped = _compose("stop", "postgres")
check(stopped.returncode == 0, "docker compose stop postgres failed")
time.sleep(2)
try:
status, body, _h = _upload(c, alice, chat, data)
check(status in (500, 503),
"compensation expected 5xx, got %s (body=%r)" % (status, body))
print(" upload при мёртвой БД вернул %s" % status)
finally:
started = _compose("start", "postgres")
check(started.returncode == 0, "docker compose start postgres failed")
_wait_ready(c, timeout=90)
after = _count_bucket_objects()
check(after == before,
"orphan-объект остался в bucket: before=%s after=%s" % (before, after))
print(" компенсация: bucket не вырос (before=%s after=%s)" % (before, after))
# ---------- параллельные загрузки с измерением RSS ----------
def test_parallel_uploads_memory(c):
"""Уровень 4: ≥10 одновременных загрузок с измерением пика RSS."""
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
barrier = threading.Barrier(10)
errors = []
def one(_):
local = Client(c.base, timeout=60)
barrier.wait()
try:
status, att, _h = _upload(local, alice, chat, data)
if status != 201:
errors.append("status=%s" % status)
except Exception as exc: # noqa: BLE001
errors.append(type(exc).__name__)
def rss():
if not API_CONTAINER:
return None
try:
out = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", API_CONTAINER],
capture_output=True, text=True, timeout=10,
)
used = out.stdout.strip().split("/")[0].strip()
if used.endswith("MiB"):
return float(used[:-3])
if used.endswith("GiB"):
return float(used[:-3]) * 1024
except Exception:
return None
return None
peak = {"value": 0.0}
stop = threading.Event()
def sampler():
while not stop.is_set():
v = rss()
if v and v > peak["value"]:
peak["value"] = v
time.sleep(0.2)
sampler_thread = threading.Thread(target=sampler, daemon=True)
sampler_thread.start()
with ThreadPoolExecutor(max_workers=10) as pool:
list(pool.map(one, range(10)))
stop.set()
sampler_thread.join(timeout=2)
check(not errors, "parallel uploads errors: %s" % errors[:3])
print(" 10 параллельных загрузок, пик RSS=%s MiB" % peak["value"])
if peak["value"]:
check(peak["value"] < 400, "peak RSS %s MiB слишком высок" % peak["value"])
# ---------- main ----------
def main():
args = arguments("Lab4 level 4 checks")
c = Client(args.base_url, args.ca_file)
print("=== oversize ===")
test_oversize(c)
print("PASS: oversize -> 413")
print("=== corrupted image ===")
test_corrupted_image(c)
print("PASS: corrupted -> 415")
print("=== spoofed MIME ===")
test_mime_spoof(c)
print("PASS: spoofed MIME -> 415")
print("=== bad filename ===")
test_bad_filename(c)
print("PASS: bad filename -> 400")
print("=== foreign attachment / cross-chat ===")
test_foreign_attachment_and_cross_chat(c)
print("PASS: foreign -> 404, cross-chat -> 404")
print("=== attachment belongs to chat ===")
test_attachment_belongs_to_chat(c)
print("PASS: attachment from another chat -> 404")
print("=== compensation on DB failure ===")
test_compensation_on_db_failure(c)
print("PASS: компенсация orphan-объектов при отказе БД")
print("=== parallel uploads memory ===")
test_parallel_uploads_memory(c)
print("PASS: 10 параллельных загрузок в пределах лимита памяти")
print("PASS: all lab4 level-4 checks passed")
if __name__ == "__main__":
run(main)
+513
View File
@@ -0,0 +1,513 @@
#!/usr/bin/env python3
"""Level 4 для lab5: варианты, outbox, идемпотентность, DLQ, lease, retry.
Запуск:
$env:PYTHONUTF8="1"
python student-tests/lab5_test_level4.py --base-url http://localhost:8080
Требует рабочий стенд из compose.yaml (LB + 2 api + worker + outbox +
RabbitMQ + MinIO + PostgreSQL + Redis). Использует psql/mc через
docker run в compose-сети, как lab4_test_level4.py.
Сценарии:
=== variants ===
Все 4 варианта: original, thumbnail (256×160), crop (128×128),
optimized (WebP). Реальные bytes и размеры, а не JSON-обещания.
=== outbox crash between COMMIT and publish ===
Остановить outbox → upload → строка outbox есть, published_at NULL
→ запустить outbox → published_at заполнено, attachment готов.
=== duplicate delivery does not create duplicates ===
Ручная повторная публикация того же job → variants не дублируются.
=== worker stopped: 202 → content 409 → ready ===
Остановить worker → upload → 202 queued → content 409 → старт
worker → ready и thumbnail доступен.
=== broken staging → retry_exhausted → failed + DLQ ===
Удалить staging-объект → worker исчерпает retry → failed
error_code=retry_exhausted + сообщение в DLQ.
=== stuck processing (lease) → queued → ready ===
Симулировать processing с истёкшим lease → recover (через s3_cleanup
или через явный SQL) возвращает в queued → worker обрабатывает → ready.
"""
import hashlib
import io
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
from image_probe import jpeg_size # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab5_default")
PG_CONTAINER = os.environ.get("PG_CONTAINER", "mtuci-postgres")
RABBIT_CONTAINER = os.environ.get("RABBIT_CONTAINER", "mtuci-rabbitmq")
MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920")
# ---------- helpers ----------
def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str):
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
req = urllib.request.Request(
url, data=data, method="POST",
headers={
"Authorization": "Bearer " + actor["token"],
"Content-Type": ctype,
},
)
try:
with c.opener.open(req, timeout=30) as resp:
return resp.code, json.loads(resp.read()), resp.headers
except urllib.error.HTTPError as e:
body = e.read()
try:
return e.code, json.loads(body), e.headers
except Exception:
return e.code, None, e.headers
def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"):
return _raw_upload(c, actor, chat["id"], filename, data, ctype)
def _env_from_file():
env = os.environ.copy()
env_file = ROOT / ".env"
if env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
env.setdefault(k.strip(), v.strip())
return env
def _psql(sql: str) -> str:
"""psql через docker compose exec в mtuci-postgres."""
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-U", "messenger", "-d", "high_load_messenger",
"-tA", "-v", "ON_ERROR_STOP=1", "-c", sql],
cwd=str(ROOT), capture_output=True, text=True, timeout=60,
)
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
return r.stdout.strip()
def _compose(action: str, service: str):
return subprocess.run(
["docker", "compose", "--env-file", ".env", action, service],
cwd=str(ROOT), capture_output=True, text=True, timeout=90,
)
def _wait_ready(c, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
return
except Failure:
time.sleep(1)
raise Failure("API не вернулся в ready за %d секунд" % timeout)
def _mc(*args: str) -> subprocess.CompletedProcess:
"""Запуск mc через одноразовый контейнер в compose-сети."""
env = _env_from_file()
user = env.get("MINIO_ROOT_USER", "minioadmin")
password = env.get("MINIO_ROOT_PASSWORD", "minioadmin")
mc_host = "http://%s:%s@minio:9000" % (user, password)
ins = subprocess.run(
["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"],
capture_output=True, text=True, timeout=10,
)
ep = (ins.stdout or "").strip()
prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"]
return subprocess.run(
["docker", "run", "--rm", "--network", COMPOSE_NETWORK,
"-e", "MC_HOST_local=" + mc_host, MC_IMAGE]
+ prefix + list(args),
capture_output=True, text=True, timeout=60,
)
def _mc_rm(key: str) -> None:
env = _env_from_file()
bucket = env.get("S3_BUCKET", "messenger-attachments")
r = _mc("rm", "local/%s/%s" % (bucket, key))
check(r.returncode == 0, "mc rm failed: %s" % r.stderr[:300])
def _dlq_count() -> int:
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
"rabbitmqctl", "list_queues", "name", "messages"],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
for line in r.stdout.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0] == "attachment.process.v1.dlq":
return int(parts[1])
return 0
def _queue_consumers(qname: str) -> int:
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
"rabbitmqctl", "list_queues", "name", "consumers"],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
for line in r.stdout.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0] == qname:
return int(parts[1])
return 0
# ---------- 1. Все 4 варианта ----------
def test_variants(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202, got %s" % status)
check(att["status"] == "queued", "upload status must be queued, got %s" % att["status"])
ready = c.wait_ready(alice, att, wait_seconds=60)
check(ready["status"] == "ready", "must be ready")
check(ready["error_code"] is None, "ready must have error_code=null")
names = sorted(v["name"] for v in ready["variants"])
check(names == ["crop", "optimized", "thumbnail"],
"expected variants thumbnail+crop+optimized, got %s" % names)
# original
download = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)
check(download["bytes"] == data, "original bytes differ")
check(download["headers"].get_content_type() == "image/png", "original content-type")
# thumbnail: JPEG 256x160, sha256, size
v = next(x for x in ready["variants"] if x["name"] == "thumbnail")
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be image/jpeg")
check(len(thumb["bytes"]) == v["size_bytes"], "thumbnail size_bytes mismatch")
check(hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], "thumbnail sha256 mismatch")
w, h = jpeg_size(thumb["bytes"])
check((w, h) == (256, 160), "thumbnail dimensions %sx%s, expected 256x160" % (w, h))
check((v["width"], v["height"]) == (256, 160), "thumbnail metadata dimensions mismatch")
# crop: JPEG 128x128
v = next(x for x in ready["variants"] if x["name"] == "crop")
crop = c.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=bob)
check(crop["headers"].get_content_type() == "image/jpeg", "crop must be image/jpeg")
check(len(crop["bytes"]) == v["size_bytes"], "crop size_bytes mismatch")
w, h = jpeg_size(crop["bytes"])
check((w, h) == (128, 128), "crop dimensions %sx%s, expected 128x128" % (w, h))
# optimized: WebP
v = next(x for x in ready["variants"] if x["name"] == "optimized")
opt = c.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=bob)
check(opt["headers"].get_content_type() == "image/webp", "optimized must be image/webp")
check(len(opt["bytes"]) == v["size_bytes"], "optimized size_bytes mismatch")
check(hashlib.sha256(opt["bytes"]).hexdigest() == v["sha256"], "optimized sha256 mismatch")
# WebP RIFF-заголовок
check(opt["bytes"][:4] == b"RIFF" and opt["bytes"][8:12] == b"WEBP", "optimized is not WebP")
# ---------- 2. Outbox: crash между COMMIT и publish ----------
def test_outbox_crash(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
# Останавливаем outbox, чтобы job застряла в БД после COMMIT.
r = _compose("stop", "outbox")
check(r.returncode == 0, "stop outbox failed: %s" % r.stderr[:300])
try:
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202 with stopped outbox, got %s" % status)
# В БД должна быть строка outbox с published_at IS NULL.
unpublished = int(_psql(
"SELECT count(*) FROM outbox WHERE published_at IS NULL"
" AND payload->>'attachment_id' = '%s';" % att["id"]
))
check(unpublished == 1,
"expected 1 unpublished outbox row for %s, got %d" % (att["id"], unpublished))
# attachment остаётся queued.
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
check(st == "queued", "attachment must stay queued while outbox stopped, got %s" % st)
finally:
r = _compose("start", "outbox")
check(r.returncode == 0, "start outbox failed: %s" % r.stderr[:300])
# После старта outbox job должна быть опубликована и обработана.
ready = c.wait_ready(alice, att, wait_seconds=90)
check(ready["status"] == "ready", "outbox recovery: attachment must become ready")
published = int(_psql(
"SELECT count(*) FROM outbox WHERE published_at IS NOT NULL"
" AND payload->>'attachment_id' = '%s';" % att["id"]
))
check(published == 1, "outbox row must be marked published, got %d" % published)
# ---------- 3. Повторная доставка не создаёт дублей ----------
def test_duplicate_delivery_no_duplicates(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202, got %s" % status)
ready = c.wait_ready(alice, att, wait_seconds=60)
check(ready["status"] == "ready", "attachment must become ready")
# Ровно 3 non-original варианта в БД.
n_before = int(_psql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
))
check(n_before == 3, "expected exactly 3 variants, got %d" % n_before)
# Повторно публикуем тот же job через ручную публикацию в RabbitMQ
# с тем же payload (эмуляция redelivery после side effect до ack).
payload_json = _psql(
"SELECT payload::text FROM outbox WHERE payload->>'attachment_id' = '%s' LIMIT 1;" % att["id"]
)
check(bool(payload_json), "payload not found in outbox")
# Публикуем в основную очередь через pika в контейнере worker.
script = (
"import os, json, pika\n"
"url = os.environ['RABBITMQ_URL']\n"
"params = pika.URLParameters(url)\n"
"conn = pika.BlockingConnection(params)\n"
"ch = conn.channel()\n"
"ch.confirm_delivery()\n"
"ch.exchange_declare('messenger.attachments', exchange_type='direct', durable=True)\n"
"body = %r\n"
"ch.basic_publish(exchange='messenger.attachments', routing_key='attachment.process.v1',\n"
" body=body.encode('utf-8'),\n"
" properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'),\n"
" mandatory=True)\n"
"conn.close()\n"
"print('PUBLISHED')\n" % payload_json
)
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "worker",
"python", "-c", script],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
check(r.returncode == 0 and "PUBLISHED" in r.stdout,
"manual duplicate publish failed: %s" % (r.stderr[:300] or r.stdout[:300]))
# Даём worker-у обработать дубль.
time.sleep(5)
n_after = int(_psql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
))
check(n_after == n_before,
"duplicate delivery created extra variants: %d -> %d" % (n_before, n_after))
# Всё ещё ready и не failed.
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
check(st == "ready", "duplicate must not change status, got %s" % st)
# ---------- 4. Остановка worker: 202 → 409 → ready ----------
def test_worker_stopped(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
r = _compose("stop", "worker")
check(r.returncode == 0, "stop worker failed: %s" % r.stderr[:300])
try:
# Upload должен вернуть 202, attachment = queued, content — 409.
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload with stopped worker must be 202, got %s" % status)
check(att["status"] == "queued", "attachment must be queued")
# content должен вернуть 409, пока не ready.
r_content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"],
(409, 404), actor=bob)
check(r_content["status"] == 409,
"content must be 409 for queued, got %s" % r_content["status"])
# API остаётся доступен для других эндпоинтов.
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
finally:
r = _compose("start", "worker")
check(r.returncode == 0, "start worker failed: %s" % r.stderr[:300])
# Ждём consumer_started и processing.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if _queue_consumers("attachment.process.v1") >= 1:
break
time.sleep(1)
else:
raise Failure("worker не подключился к очереди за 30 секунд")
ready = c.wait_ready(alice, att, wait_seconds=90)
check(ready["status"] == "ready", "after worker start attachment must become ready")
# thumbnail доступен
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail content-type")
# ---------- 5. Broken staging → retry_exhausted → failed + DLQ ----------
def test_broken_staging_dlq(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
# Останавливаем worker, чтобы сначала зафиксировать staging_key в БД,
# затем удалить staging-объект и запустить worker.
r = _compose("stop", "worker")
check(r.returncode == 0, "stop worker failed")
try:
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202")
# Удаляем staging-объект из S3, чтобы worker не смог его прочитать.
staging = _psql(
"SELECT object_key FROM attachments WHERE id = '%s';" % att["id"]
)
check(staging.startswith("staging/"),
"queued attachment object_key must be staging/*, got %r" % staging)
_mc_rm(staging)
finally:
r = _compose("start", "worker")
check(r.returncode == 0, "start worker failed")
dlq_before = _dlq_count()
# Ждём финального failed (retry: 5+15+60 ≈ 80c + обработка).
deadline = time.monotonic() + 180
final_status = None
error_code = None
while time.monotonic() < deadline:
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
if st == "failed":
final_status = st
error_code = _psql(
"SELECT coalesce(error_code, '') FROM attachments WHERE id = '%s';" % att["id"]
)
break
if st == "ready":
raise Failure("attachment unexpectedly became ready: staging was deleted")
time.sleep(5)
check(final_status == "failed",
"expected failed after retry exhaustion, got %s" % final_status)
check(error_code == "retry_exhausted",
"expected error_code=retry_exhausted, got %r" % error_code)
# DLQ должен содержать хотя бы одно сообщение.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if _dlq_count() > dlq_before:
break
time.sleep(1)
else:
raise Failure("DLQ did not receive the message after retry exhaustion "
"(dlq_before=%d)" % dlq_before)
# ---------- 6. Stuck processing (lease) → queued → ready ----------
def test_stuck_processing_recovered(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
# Останавливаем worker, чтобы job осталась queued и мы могли её «сломать».
r = _compose("stop", "worker")
check(r.returncode == 0, "stop worker failed")
try:
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202, got %s" % status)
# Симулируем зависший processing: status=processing, lease в прошлом.
_psql(
"UPDATE attachments SET status='processing',"
" processing_until = now() - interval '5 minutes',"
" updated_at = now() WHERE id = '%s';" % att["id"]
)
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
check(st == "processing", "setup failed, expected processing, got %s" % st)
finally:
r = _compose("start", "worker")
check(r.returncode == 0, "start worker failed")
# Проверяем, что SQL-recover (эквивалент recover_stuck_processing)
# переводит запись в queued. Это и есть логика, которую вызывает
# s3_cleanup по интервалу. Здесь дёргаем явно, чтобы не зависеть от
# CLEANUP_INTERVAL_SECONDS.
recovered = int(_psql(
"WITH upd AS ("
" UPDATE attachments SET status='queued', processing_until=NULL, updated_at=now()"
" WHERE status='processing' AND processing_until IS NOT NULL"
" AND processing_until < now()"
" RETURNING id"
") SELECT count(*) FROM upd;"
))
check(recovered >= 1, "recover_stuck_processing did not return any row")
# worker после старта подхватит queued и доведёт до ready.
ready = c.wait_ready(alice, att, wait_seconds=90)
check(ready["status"] == "ready", "recovered attachment must become ready")
# ---------- main ----------
def main():
args = arguments("Lab5 level 4 checks")
c = Client(args.base_url, args.ca_file)
c.call("GET", "/health/ready", schema="Health")
print("=== variants (thumbnail/crop/optimized) ===")
test_variants(c)
print("PASS: original, thumbnail 256×160, crop 128×128, optimized WebP")
print("=== outbox crash between COMMIT and publish ===")
test_outbox_crash(c)
print("PASS: outbox recovers job after stop/start without manual intervention")
print("=== duplicate delivery does not create duplicates ===")
test_duplicate_delivery_no_duplicates(c)
print("PASS: re-delivery of the same job is a no-op")
print("=== worker stopped: 202 → content 409 → ready ===")
test_worker_stopped(c)
print("PASS: accepted work survives worker restart")
print("=== broken staging → retry_exhausted → failed + DLQ ===")
test_broken_staging_dlq(c)
print("PASS: bounded retry, failed with safe error_code, message in DLQ")
print("=== stuck processing (lease) → queued → ready ===")
test_stuck_processing_recovered(c)
print("PASS: expired lease is recovered, same attachment becomes ready")
print("PASS: all lab5 level-4 checks passed")
if __name__ == "__main__":
run(main)
+597
View File
@@ -0,0 +1,597 @@
#!/usr/bin/env python3
"""Level 4 для lab6: сквозной trace API→broker→worker, метрики, логи, dashboard,
alert rules, поведение при отказе telemetry-backend.
Запуск:
$env:PYTHONUTF8="1"
python student-tests/lab6_test_level4.py --base-url http://localhost:8080
"""
import base64
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost:3000")
GRAFANA_ADMIN_USER = os.environ.get("GRAFANA_ADMIN_USER", "admin")
GRAFANA_ADMIN_PASSWORD = os.environ.get("GRAFANA_ADMIN_PASSWORD", "admin")
READY_TIMEOUT = int(os.environ.get("READY_TIMEOUT", "180"))
def _compose_exec(service: str, *args: str, timeout: int = 30):
return subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", service, *args],
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
)
def _compose(action: str, service: str, timeout: int = 90):
return subprocess.run(
["docker", "compose", "--env-file", ".env", action, service],
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
)
def _compose_logs(service: str, tail: int = 20) -> str:
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "logs", "--no-color",
"--tail=%d" % tail, service],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
return (r.stdout or "") + (r.stderr or "")
def _http_get(url: str, timeout: float = 10.0, basic_auth: tuple | None = None):
headers = {"Accept": "application/json"}
if basic_auth is not None:
token = base64.b64encode(("%s:%s" % basic_auth).encode()).decode()
headers["Authorization"] = "Basic " + token
req = urllib.request.Request(url, headers=headers, method="GET")
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode("utf-8", "replace")
except urllib.error.HTTPError as e:
return e.code, e.read().decode("utf-8", "replace")
except (urllib.error.URLError, OSError) as e:
return 0, "%s: %s" % (type(e).__name__, e)
def _prom_query(query: str) -> dict:
qs = "query=" + urllib.parse.quote(query, safe="")
r = _compose_exec(
"prometheus", "sh", "-c",
"wget -qO- 'http://localhost:9090/api/v1/query?%s'" % qs,
)
if r.returncode != 0 or not r.stdout.strip():
raise Failure("prometheus query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
try:
return json.loads(r.stdout)
except Exception:
raise Failure("prometheus non-JSON: %s" % r.stdout[:300])
def _tempo_search(query: str, since_seconds: int = 300, limit: int = 20) -> list[dict]:
now = int(time.time())
start = now - since_seconds
qs = "q=%s&start=%d&end=%d&limit=%d" % (
urllib.parse.quote(query, safe=""), start, now, limit,
)
r = _compose_exec(
"tempo", "sh", "-c",
"wget -qO- 'http://localhost:3200/api/search?%s'" % qs,
)
if r.returncode != 0 or not r.stdout.strip():
return []
try:
data = json.loads(r.stdout)
except Exception:
return []
return data.get("traces", []) or []
def _tempo_get_trace(trace_id: str) -> dict:
r = _compose_exec(
"tempo", "sh", "-c",
"wget -qO- 'http://localhost:3200/api/traces/%s'" % trace_id,
)
if r.returncode != 0 or not r.stdout.strip():
raise Failure("tempo trace fetch failed: %s" % (r.stderr[:300] or r.stdout[:300]))
return json.loads(r.stdout)
def _tempo_span_names(trace: dict) -> set[str]:
names: set[str] = set()
for batch in trace.get("batches", []):
scopes = batch.get("scopeSpans") or batch.get("instrumentationLibrarySpans") or []
for scope in scopes:
for span in scope.get("spans", []) or []:
names.add(span.get("name", ""))
return names
def _loki_query(query: str, since_seconds: int = 300, limit: int = 200) -> dict:
end = int(time.time() * 1e9)
start = end - since_seconds * 1_000_000_000
qs = "query=%s&start=%d&end=%d&limit=%d" % (
urllib.parse.quote(query, safe=""), start, end, limit,
)
r = _compose_exec(
"loki", "sh", "-c",
"wget -qO- 'http://localhost:3100/loki/api/v1/query_range?%s'" % qs,
)
if r.returncode != 0 or not r.stdout.strip():
raise Failure("loki query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
try:
return json.loads(r.stdout)
except Exception:
raise Failure("loki returned non-JSON: %s" % r.stdout[:300])
def _raw_upload(c: Client, actor, chat_id: str, filename: str,
data: bytes, ctype: str):
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
req = urllib.request.Request(
url, data=data, method="POST",
headers={
"Authorization": "Bearer " + actor["token"],
"Content-Type": ctype,
},
)
try:
with c.opener.open(req, timeout=30) as resp:
return resp.code, json.loads(resp.read()), resp.headers
except urllib.error.HTTPError as e:
body = e.read()
try:
return e.code, json.loads(body), e.headers
except Exception:
return e.code, None, e.headers
def _wait_ready(c, actor, att, timeout: int) -> dict:
"""Обёртка с диагностикой: если долго не ready — показываем статус и логи."""
deadline = time.monotonic() + timeout
last_status = None
while time.monotonic() < deadline:
a = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor,
schema="Attachment")["json"]
last_status = a.get("status")
if last_status == "ready":
return a
if last_status == "failed":
raise Failure("attachment failed: error_code=%r" % a.get("error_code"))
time.sleep(0.5)
diag = (
"\nlast_status=%r (waited %ss)\n--- outbox logs ---\n%s\n--- worker logs ---\n%s"
% (last_status, timeout, _compose_logs("outbox", 20),
_compose_logs("worker", 20))
)
raise Failure("attachment did not become ready" + diag)
def _wait_trace_with_spans(since_seconds: int, need_substrings: list[str],
timeout: float = 180.0) -> dict:
deadline = time.monotonic() + timeout
last_seen: list[dict] = []
while time.monotonic() < deadline:
for svc in ("messenger-api", "messenger-worker", "messenger-outbox"):
candidates = _tempo_search(
'{resource.service.name = "%s"}' % svc,
since_seconds=since_seconds,
)
for t in candidates:
trace_id = t.get("traceID")
if not trace_id:
continue
try:
full = _tempo_get_trace(trace_id)
except Failure:
continue
names = _tempo_span_names(full)
last_seen.append({"trace_id": trace_id, "names": sorted(names)})
if all(any(s in n for n in names) for s in need_substrings):
return {"trace_id": trace_id, "names": sorted(names)}
time.sleep(3)
tail = "; ".join(
"%s -> %s" % (x["trace_id"][:8], x["names"][:6]) for x in last_seen[:5]
)
raise Failure(
"не дождались trace со span-names, содержащими %s за %ss; последние trace: %s"
% (need_substrings, timeout, tail or "нет")
)
def test_trace_api_to_worker(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
started = int(time.time())
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
check(status == 202, "upload must be 202, got %s" % status)
check(att["status"] == "queued", "upload status must be queued")
ready = _wait_ready(c, alice, att, timeout=READY_TIMEOUT)
check(ready["status"] == "ready", "attachment must become ready")
since = max(60, int(time.time()) - started + 180)
trace = _wait_trace_with_spans(
since_seconds=since,
need_substrings=[
"/api/v1/chats/{chat_id}/attachments",
"job.publish",
"job.consume",
],
timeout=180,
)
print(" trace_id:", trace["trace_id"])
print(" spans:", ", ".join(trace["names"]))
def test_metrics_in_prometheus(c):
alice, bob, eve, chat = c.bootstrap()
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
data = SAMPLE.read_bytes()
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
check(status == 202, "upload must be 202")
_wait_ready(c, alice, att, timeout=READY_TIMEOUT)
deadline = time.monotonic() + 120
required = {
"http_requests_total",
"http_request_duration_seconds_bucket",
"jobs_processed_total",
"job_processing_duration_seconds_bucket",
}
got: set[str] = set()
while time.monotonic() < deadline:
got = set()
for name in required:
r = _prom_query("count(%s)" % name)
series = r.get("data", {}).get("result", [])
if series:
got.add(name)
if got == required:
break
time.sleep(3)
missing = required - got
check(not missing, "Prometheus missing metrics: %s" % sorted(missing))
print(" metrics in Prometheus:", sorted(got))
r = _prom_query("count by (http_route, http_status_class) (http_requests_total)")
series = r.get("data", {}).get("result", [])
check(series, "http_requests_total has no series")
import re as _re
for s in series:
metric = s.get("metric", {})
route = metric.get("http_route", "")
check(route.startswith("/api/") or route.startswith("/health"),
"unexpected http_route: %r" % route)
check(_re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
route) is None,
"http_route contains raw UUID: %r" % route)
r = _prom_query("sum(jobs_processed_total{outcome=\"success\"})")
series = r.get("data", {}).get("result", [])
check(series, "jobs_processed_total{outcome=success} absent")
value = float(series[0]["value"][1])
check(value >= 1, "jobs success metric is %r" % value)
print(" jobs_processed_total{outcome=success} =", value)
def test_logs_in_loki(c):
time.sleep(15)
deadline = time.monotonic() + 90
streams = []
while time.monotonic() < deadline:
r = _loki_query('{service_name="messenger-api"} |= "event=http"',
since_seconds=300)
streams = r.get("data", {}).get("result", [])
if streams:
break
time.sleep(3)
check(streams, "Loki: no messenger-api logs with event=http")
lines = [v[1] for s in streams for v in s.get("values", [])]
check(any("request_id=" in line for line in lines),
"API logs missing request_id")
r = _loki_query('{service_name="messenger-worker"} |= "event="', since_seconds=300)
check(r.get("data", {}).get("result", []),
"Loki: no messenger-worker logs")
# Ищем запись event=http с ненулевым trace_id (корреляция лог↔trace).
deadline = time.monotonic() + 90
ok = False
while time.monotonic() < deadline:
r = _loki_query(
'{service_name=~"messenger-api|messenger-worker"} '
'|~ "event=(http|job_ready)" |~ "trace_id=" ',
since_seconds=300,
)
streams = r.get("data", {}).get("result", [])
for s in streams:
for v in s.get("values", []):
line = v[1]
idx = line.find("trace_id=")
if idx < 0:
continue
tail = line[idx + len("trace_id="):]
tid = tail.split()[0].strip()
if len(tid) == 32 and set(tid) != {"0"}:
ok = True
break
if ok:
break
if ok:
break
time.sleep(3)
check(ok, "Loki: no logs with non-zero trace_id (log↔trace correlation)")
print(" Loki OK: api+worker logs with non-zero trace_id found")
def _grafana_ds_uids(auth) -> dict[str, dict]:
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
check(code == 200, "grafana datasources API %s" % code)
return {d.get("uid"): d for d in json.loads(body)}
def test_grafana_provisioning(c):
deadline = time.monotonic() + 60
auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD)
while time.monotonic() < deadline:
code, _ = _http_get(GRAFANA_URL + "/api/health")
if code == 200:
break
time.sleep(2)
else:
raise Failure("Grafana API не поднялась за 60 с")
by_uid = _grafana_ds_uids(auth)
# Требуемые UID явно заданы в provisioning и совпадают со ссылками в
# dashboard/rules/derived fields.
for uid, type_ in (("prometheus", "prometheus"),
("loki", "loki"),
("tempo", "tempo")):
check(uid in by_uid, "Grafana missing datasource uid=%r (have %s)"
% (uid, sorted(by_uid)))
check(by_uid[uid].get("type") == type_,
"datasource uid=%r has wrong type: %r"
% (uid, by_uid[uid].get("type")))
print(" datasources by uid:", sorted(by_uid))
code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6",
basic_auth=auth)
check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code)
dash = json.loads(body)
dashboard = dash.get("dashboard", {})
panels = dashboard.get("panels", [])
titles = {p.get("title") for p in panels}
check(any("p95" in (t or "") for t in titles),
"dashboard missing p95 panel")
check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles),
"dashboard missing jobs panel")
# Проверяем, что панель логов ссылается на существующий uid Loki,
# а не на «магическое» имя.
for p in panels:
ds = p.get("datasource") or {}
uid = ds.get("uid")
if uid is not None:
check(uid in by_uid,
"panel %r references unknown datasource uid %r"
% (p.get("title"), uid))
print(" dashboard panels:", sorted(t for t in titles if t))
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
basic_auth=auth)
check(code == 200, "grafana alert-rules API %s" % code)
rules = json.loads(body)
check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules))
for r in rules:
# datasourceUid в data должен существовать.
for item in r.get("data", []):
uid = item.get("datasourceUid")
if uid and uid != "__expr__":
check(uid in by_uid,
"rule %r references unknown datasource uid %r"
% (r.get("title"), uid))
# Все правила должны быть health=ok. Если datasources не согласованы,
# Grafana вернёт health=error "data source not found", как было
# в замечании проверяющего.
deadline = time.monotonic() + 180
while True:
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
basic_auth=auth)
check(code == 200, "grafana alert-rules API %s" % code)
rules = json.loads(body)
bad = [r.get("title") for r in rules
if r.get("health") not in (None, "ok")]
if not bad:
break
if time.monotonic() > deadline:
raise Failure("Grafana alert rules with non-ok health: %s" % bad)
time.sleep(3)
print(" alert rules:", [r.get("title") for r in rules])
def _prom_alerts() -> list[dict]:
r = _compose_exec("prometheus", "sh", "-c",
"wget -qO- 'http://localhost:9090/api/v1/alerts'")
if r.returncode != 0 or not r.stdout.strip():
return []
try:
data = json.loads(r.stdout)
except Exception:
return []
return data.get("data", {}).get("alerts", []) or []
def _wait_alert(name: str, state: str, timeout: float = 240.0):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
alerts = _prom_alerts()
# Prometheus /api/v1/alerts возвращает только pending/firing.
# Состояние "inactive" == алерта с таким alertname нет в списке.
active = [
a for a in alerts
if a.get("labels", {}).get("alertname") == name
]
if state == "inactive":
if not active:
return
else:
for a in active:
if a.get("state") == state:
return
time.sleep(3)
raise Failure("alert %s never reached %s" % (name, state))
def test_alert_firing_and_resolved(c):
alice, bob, eve, chat = c.bootstrap()
stopped = _compose("stop", "postgres")
check(stopped.returncode == 0, "stop postgres failed")
try:
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call("GET", "/api/v1/users/me", (401, 503), actor=alice)
except Failure:
pass
time.sleep(0.3)
_wait_alert("MessengerHighErrorRate", "firing", timeout=240)
print(" MessengerHighErrorRate firing")
finally:
started = _compose("start", "postgres")
check(started.returncode == 0, "start postgres failed")
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
break
except Failure:
time.sleep(1)
else:
raise Failure("API didn't become ready after postgres restart")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
except Failure:
pass
time.sleep(0.3)
_wait_alert("MessengerHighErrorRate", "inactive", timeout=300)
print(" MessengerHighErrorRate resolved")
def test_telemetry_backend_down(c):
for svc in ("tempo", "loki", "prometheus"):
_compose("start", svc)
time.sleep(3)
stopped = {}
for svc in ("prometheus", "loki", "tempo"):
r = _compose("stop", svc)
check(r.returncode == 0, "stop %s failed: %s" % (svc, r.stderr[:300]))
stopped[svc] = True
try:
alice, bob, eve, chat = c.bootstrap()
deadline = time.monotonic() + 30
ok = 0
while time.monotonic() < deadline:
try:
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
c.message(alice, chat, "telem-down-" + uuid.uuid4().hex[:8])
ok += 1
except Failure:
pass
time.sleep(0.2)
check(ok >= 5, "API unavailable while telemetry backend is down (ok=%d)" % ok)
# В образе otel/opentelemetry-collector-contrib нет coreutils `true`,
# поэтому проверяем живость штатным бинарником самого коллектора
# (тот же приём, что в healthcheck compose.yaml).
ps = _compose_exec("otel-collector", "/otelcol-contrib", "--version")
check(ps.returncode == 0,
"otel-collector died while backends were unavailable")
stats = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}",
"mtuci-otel-collector"],
capture_output=True, text=True, timeout=15,
)
if stats.returncode == 0:
used = stats.stdout.strip().split("/")[0].strip()
mb = None
if used.endswith("MiB"):
mb = float(used[:-3])
elif used.endswith("GiB"):
mb = float(used[:-3]) * 1024
if mb is not None:
check(mb < 260, "otel-collector RSS=%s MiB exceeds limit" % mb)
print(" otel-collector RSS while backends down:", mb, "MiB")
finally:
for svc in stopped:
_compose("start", svc)
time.sleep(3)
print(" API and Collector remained healthy while backends were down")
def main():
args = arguments("Lab6 level 4 checks")
c = Client(args.base_url, args.ca_file)
c.call("GET", "/health/ready", schema="Health")
print("=== trace API -> broker -> worker (Tempo) ===")
test_trace_api_to_worker(c)
print("PASS: single trace with upload + job.publish + job.consume")
print("=== metrics in Prometheus ===")
test_metrics_in_prometheus(c)
print("PASS: http_requests_total + duration + jobs_processed_total present, "
"low-cardinality route labels")
print("=== logs in Loki ===")
test_logs_in_loki(c)
print("PASS: structured logs of api+worker with trace_id")
print("=== Grafana provisioning ===")
test_grafana_provisioning(c)
print("PASS: dashboard messenger-lab6 + >=2 alert rules provisioned")
print("=== alert firing and resolved ===")
test_alert_firing_and_resolved(c)
print("PASS: alert fires on artificial outage and resolves on recovery")
print("=== telemetry backend down ===")
test_telemetry_backend_down(c)
print("PASS: API works, Collector stays within memory limit")
print("PASS: all lab6 level-4 checks passed")
if __name__ == "__main__":
run(main)