diff --git a/.env.example b/.env.example index 2b32655..5e7fad3 100644 --- a/.env.example +++ b/.env.example @@ -5,19 +5,40 @@ APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +DATABASE_URL=postgresql://user:password@postgres:5432/high_load_messenger +POSTGRES_USER=user +POSTGRES_PASSWORD=password +POSTGRES_DB=high_load_messenger +# Redis. Пароль не задан: Redis доступен только внутри compose-сети (порт наружу не публикуется) +REDIS_URL=redis://redis:6379/0 +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 + +# --- MinIO / S3 (lab4) --- +# Внутренний endpoint, доступен только из compose-сети. +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Администратор MinIO (используется только minio и minio-init). +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=minioadmin-secret-change-me +# Прикладные S3-credentials (ограниченные права). Используются API и cleanup. +S3_APP_KEY_ID=messenger-app +S3_APP_SECRET_KEY=messenger-app-secret-change-me +# Лимиты (контракт + lab4) MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Фоновая сверка S3↔БД (level 4) +CLEANUP_INTERVAL_SECONDS=300 +# Grace period: объекты моложе N секунд не удаляются (защита in-flight uploads). +ORPHAN_GRACE_SECONDS=3600 + +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=messenger-rabbit-change-me +RABBITMQ_URL=amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F + BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 @@ -25,6 +46,9 @@ JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 OTEL_SERVICE_NAME=messenger-api # Worker must set a distinct service.name. + +OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_ADMIN_PASSWORD=admin \ No newline at end of file diff --git a/.gitignore b/.gitignore index b5a7189..9c71b71 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ certs/private/ *.jks *.pem *.tfstate* +*.dump volumes/ data/ backups/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b261122 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# --- build stage --- +FROM python:3.12-slim AS builder + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir --prefix=/install -r requirements.txt + +# --- runtime stage --- +FROM python:3.12-slim + +# Непривилегированный пользователь +RUN useradd --create-home --shell /bin/bash appuser + +WORKDIR /app +COPY --from=builder /install /usr/local +COPY src/ ./src/ +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/app/src + +USER appuser +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live').read()" + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--app-dir", "src"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d17..512c8fd 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-student load test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; test-student: own tests; load: synthetic writes" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +18,9 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-student: + $(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)" + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index b97ae42..c758d3b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,283 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Васюкова Наталья, БВТ2303 +- Целевая оценка: 4 +- Commit/PR: solution/BVT2303-Vasyukova +- Использованные источники, библиотеки и помощь ИИ: + - Контракт: contracts/openapi.json, contracts/README.md, contracts/observability.md + - Инструкции: README.md, COURSE.md, CONTRIBUTING.md + - Библиотеки: fastapi==0.115.0, uvicorn[standard]==0.32.0, psycopg[binary]==3.2.3, psycopg_pool==3.2.3, redis==5.2.0, boto3==1.35.74, Pillow==11.0.0, pika==1.3.2, opentelemetry-api==1.28.2, opentelemetry-sdk==1.28.2, opentelemetry-exporter-otlp-proto-http==1.28.2, opentelemetry-instrumentation-fastapi==0.49b2, opentelemetry-instrumentation-requests==0.49b2 + - ИИ: DeepSeek — помощь в структурировании кода и отчёта. Весь код проверен, отредактирован и запущен разработчиком. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Версии runtime/образов: +- Python: 3.12 (образ `python:3.12-slim`) +- FastAPI 0.115.0, uvicorn[standard] 0.32.0, psycopg[binary] 3.2.3, psycopg_pool 3.2.3, redis 5.2.0, boto3 1.35.74, Pillow 11.0.0, pika 1.3.2 +- OTel SDK 1.28.2, FastAPI instrumentation 0.49b2 +- PostgreSQL 18.6-alpine, Redis 7.4.1-alpine, nginx 1.27.3-alpine +- RabbitMQ 3.13.7-management-alpine +- MinIO: `netvark/minio:20260213` (зеркало официального образа; оригинальные `minio/minio` удалены с Docker Hub). CLI `mc`: `netvark/mc:20260920`. +- OTel Collector: `otel/opentelemetry-collector-contrib:0.108.0` +- Prometheus: `prom/prometheus:v2.54.1`, Loki: `grafana/loki:3.1.1`, Tempo: `grafana/tempo:2.5.0`, Grafana: `grafana/grafana:11.2.0` +- Docker Desktop (Windows) + +### Запуск стека + +1. Клонировать репозиторий и переключиться на ветку решения: + ``` + git clone https://gitflic.ru/project/vasyukova-nat/mtusi2026lab5.git + git checkout solution/BVT2303-Vasyukova + ``` +2. Создать `.env` из шаблона: + ``` + copy .env.example .env + ``` +3. Собрать и поднять (LB + 2 реплики API + PostgreSQL + Redis + MinIO + minio-init + rabbitmq + worker + outbox + s3_cleanup + OTel Collector + Prometheus + Loki + Tempo + Grafana): + ``` + docker compose --env-file .env up -d --build + ``` +4. Проверить готовность: + ``` + curl.exe -s http://localhost:8080/health/ready + ``` + Ожидается: `{"status":"ok"}`. +5. Grafana доступна на `http://localhost:3000` (admin/admin). Дашборд `messenger-lab6` и datasources импортируются автоматически. + +Все проверки ниже выполнялись с `PYTHONUTF8=1` (Windows по умолчанию использует cp1251 и падает на чтении UTF-8 JSON): +``` +$env:PYTHONUTF8="1" +``` + +### Проверки + +`make` в среде автора недоступен (Windows без GNU Make), поэтому использованы эквивалентные прямые вызовы скриптов. Для Linux/CI рядом указаны команды `make`. + +| Makefile-команда | Прямой вызов | +| --- | --- | +| `make check` | `python scripts/check_contract.py` | +| `make validate` | `python scripts/validate_openapi.py` | +| `make test` | `python tests/smoke.py --base-url http://localhost:8080` | +| `make test-persistence` | `python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8080` | +| `make test-failover` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8080` | +| `make test-student` | `python student-tests/lab6_test_level4.py --base-url http://localhost:8080` | + +### Остановка и удаление учебных данных + +``` +docker compose stop # данные PostgreSQL, Redis, MinIO, RabbitMQ, Prometheus, Loki, Tempo, Grafana сохраняются (volumes) +docker compose down # volumes сохраняются +docker compose down -v # ТОЛЬКО если осознанно стираем данные +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Схема процессов и хранилищ: + +``` + ┌─ api-1 (uvicorn:8080) ┐ +HTTP-клиент → lb:80 ──┤ │ + (nginx) └─ api-2 (uvicorn:8080) ├→ PostgreSQL 18.6 (метаданные, outbox) + ├→ Redis 7.4.1 (сессии) + ├→ MinIO (S3: staging + final байты) + └→ RabbitMQ (durable job queue) + +outbox (publisher) → читает unpublished из PostgreSQL → публикует в RabbitMQ +worker (consumer) → забирает job из RabbitMQ → читает staging → пишет final+variants → status=ready +s3_cleanup → сверка S3↔БД, recover зависших processing + +API/worker/outbox → OTLP/HTTP → otel-collector → ┬→ Tempo (traces) + ├→ Loki (logs) + └→ Prometheus (metrics) + → Grafana (дашборд, алерты) +``` + +Все сервисы — в одной compose-сети `mtuci2026lab6_default`. Публично опубликован только порт LB `127.0.0.1:8080:80` и Grafana `127.0.0.1:3000:3000`. Реплики `api` **не имеют `ports`** — трафик к ним идёт исключительно через LB. PostgreSQL, Redis, MinIO, RabbitMQ, OTel Collector (4317/4318/8888/8889), Prometheus (9090), Loki (3100), Tempo (3200) доступны только по внутренним именам. + +Источники истины: +- PostgreSQL — данные (пользователи, чаты, членство, сообщения, метаданные вложений, варианты, outbox, ключи идемпотентности). +- Redis — серверные сессии (общие для всех реплик). +- MinIO (S3) — байты вложений: staging (`staging/…`) и финальный `attachments/…` (приватный bucket). +- RabbitMQ — durable очередь job'ов. Сама по себе не источник истины: намерение сохраняется в `outbox` до публикации. +- Tempo/Loki/Prometheus — telemetry storage. Их падение не останавливает API. + +Схема БД (подробнее — `src/app/storage/db.py`, константа `SCHEMA_SQL`): +``` +users(id UUID PK, username TEXT UNIQUE, display_name TEXT, password_hash TEXT, created_at TIMESTAMPTZ) +chats(id UUID PK, title TEXT, created_at TIMESTAMPTZ) +chat_members(chat_id FK→chats, user_id FK→users, PK(chat_id,user_id)) +messages(id UUID PK, chat_id FK→chats, sender_id FK→users, text TEXT, created_at TIMESTAMPTZ) + + INDEX idx_messages_chat_created(chat_id, created_at, id) +sessions(token TEXT PK, user_id FK→users, expires_at TIMESTAMPTZ) -- fallback SESSION_STORE=postgres +idempotency_keys(user_id FK→users, method TEXT, path TEXT, key TEXT, + body_hash TEXT, response_status INT, response_body JSONB, created_at TIMESTAMPTZ, + PRIMARY KEY (user_id, method, path, key)) + + INDEX idx_idempotency_created(created_at) +attachments(id UUID PK, chat_id FK→chats, owner_id FK→users, + filename TEXT, content_type TEXT, size_bytes BIGINT, sha256 TEXT, + object_key TEXT UNIQUE, status TEXT, error_code TEXT, + processing_until TIMESTAMPTZ, + created_at TIMESTAMPTZ, updated_at TIMESTAMPTZ) + + INDEX idx_attachments_chat(chat_id) +message_attachments(message_id FK→messages, attachment_id FK→attachments, position INT, + PRIMARY KEY (message_id, attachment_id)) +attachment_variants(attachment_id FK→attachments, pipeline_version INT, variant TEXT, + object_key TEXT, content_type TEXT, size_bytes BIGINT, + width INT, height INT, sha256 TEXT, created_at TIMESTAMPTZ, + PRIMARY KEY (attachment_id, pipeline_version, variant)) +outbox(id BIGSERIAL PK, job_id UUID UNIQUE, payload JSONB, created_at TIMESTAMPTZ, + published_at TIMESTAMPTZ) + + PARTIAL INDEX idx_outbox_unpublished(id) WHERE published_at IS NULL +``` + +Применяется идемпотентно через `init_schema()` в отдельном one-shot сервисе `migrate` (`python -m app.storage.db`). Все временные метки — `TIMESTAMPTZ`. + +Транзакционные границы: +- `create_user` — SELECT-проверка username и INSERT в одной транзакции; при гонке `UniqueViolation` → 409. +- `create_chat` — INSERT `chats` + INSERT `chat_members` в одной транзакции, откат целиком. +- `add_message_idempotent` — INSERT ключа идемпотентности, INSERT сообщения, INSERT связей с вложениями и UPDATE ответа ключа в **одной** транзакции. +- `create_attachment_queued` — INSERT `attachments` (status=`queued`) + INSERT `outbox` в **одной** транзакции. Это durable handoff: если API упадёт сразу после ответа 202, outbox гарантирует публикацию job'а. +- `finalize_ready` — UPDATE `attachments` (status=`ready`, `object_key`=final) + INSERT `attachment_variants` с `ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING` в **одной** транзакции. Идемпотентно к повторной доставке. + +### Дерево проекта (изменения относительно lab5 отмечены `*`) + +``` +mtuci2026lab6/ +├── compose.yaml * # + otel-collector, prometheus, loki, tempo, grafana; tag lab6 +├── Dockerfile +├── docker-entrypoint.sh # генерирует INSTANCE_ID, пишет /tmp/instance_id +├── infra/ +│ ├── nginx.conf +│ ├── otel/collector.yaml * # OTLP receivers, memory_limiter, batch, exporters +│ ├── prometheus/ +│ │ ├── prometheus.yaml * # scrape otel-collector:8889 +│ │ └── rules/alerts.yaml * +│ ├── loki/loki.yaml * +│ ├── tempo/tempo.yaml * +│ ├── grafana/provisioning/ +│ │ ├── datasources/datasources.yaml * # Prometheus/Loki/Tempo +│ │ ├── dashboards/provider.yaml * +│ │ ├── dashboards/json/messenger.json * +│ │ └── alerting/{contactpoints,policies,rules}.yaml * +│ └── runbooks/ * # high-error-rate.md, queue-lag.md, slow-upload.md +├── Makefile +├── requirements.txt * # + opentelemetry-* +├── src/app/ +│ ├── main.py * # OTel init, HTTP-метрики, X-Request-Id/X-Instance-Id +│ ├── telemetry.py * # новый: единая точка init OTel + лог-формат +│ ├── outbox_worker.py * # trace context из outbox payload → job.publish span +│ ├── worker.py * # consumer span job.consume, метрики jobs_* +│ ├── api/attachments.py * # inject traceparent в outbox payload +│ ├── api/... # без изменений с lab5 +│ ├── models/schemas.py +│ └── storage/ # без изменений с lab5 +├── scripts/ +│ ├── restart.sh / .bat +│ ├── stop-one.sh / .bat +│ └── rolling-update.sh +└── student-tests/ + └── lab6_test_level4.py * # новый: traces, metrics, logs, dashboard, alerts, telemetry down +``` + +### Что нового в lab6 относительно lab5 + +- **OTel SDK** в API, worker и outbox: `init_telemetry()` настраивает TracerProvider, MeterProvider, LoggerProvider; OTLP/HTTP → `otel-collector:4318`. +- **Resource attributes**: `service.name` (messenger-api / messenger-worker / messenger-outbox), `service.version=1.6.0`, `service.instance.id`, `deployment.environment.name=lab`. +- **Логи**: единый формат с `trace_id=`/`span_id=`; `TraceContextFilter` подставляет их из активного span и в stdout, и в OTLP-экспортёр → Loki. +- **Метрики**: `http_requests_total{http.method,http.route,http.status_class}`, `http_request_duration_seconds_bucket`, `jobs_processed_total{job.type,outcome}`, `job_processing_duration_seconds_bucket`. +- **Трассы**: server span FastAPI + собственный `http.request` span в middleware; `job.publish` (PRODUCER) в outbox; `job.consume` (CONSUMER) в worker. W3C traceparent инжектится API → outbox payload → RabbitMQ headers → worker extract. +- **Collector**: `memory_limiter` (192 MiB + spike 64), `batch`, `resource` (upsert deployment.environment.name); exporters otlphttp/tempo, otlphttp/loki, prometheus:8889. Retry ограничен (`max_elapsed_time=60s`), sending_queue `queue_size=1000`. +- **Grafana provisioning**: datasources Prometheus/Loki/Tempo, dashboard `messenger-lab6`, alert rules `MessengerHighErrorRate` + `MessengerJobsBacklogGrowing` (+ `MessengerUploadLatencyHigh`). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда | Наблюдаемый результат | Уровень | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python tests/smoke.py` | PASS: lab6 public API scenario | 3 | +| Collector + хранилища 3 сигналов + Grafana в общем стенде | `docker compose ps` | otel-collector, prometheus, loki, tempo, grafana Up | 3 | +| API/worker отдают OTLP в Collector | Grafana → Explore → Prometheus/Tempo/Loki | метрики `http_*`, `jobs_*`; реальные traces; логи `messenger-api/worker` | 3 | +| Импортируемый dashboard | `test_grafana_provisioning` | `/api/dashboards/uid/messenger-lab6` → 200, панели RPS/errors/p95/jobs | 3 | +| Логи с trace_id | `lab6_test_level4.py::test_logs_in_loki` | ненулевой `trace_id=` в логах api+worker | 3/4 | +| Сохранение метаданных и сессии при рестарте | `resilience.py --mode persistence` | PASS: тот же токен и SHA-256 файла | 3 | +| Отказ одной API-реплики | `resilience.py --mode failover` | observed_instances_before=2, failed_attempts=0 | 3 | +| Сквозной trace API → broker → worker | `lab6_test_level4.py::test_trace_api_to_worker` | один trace со span'ами `POST /api/v1/chats/{chat_id}/attachments`, `job.publish`, `job.consume` | 4 | +| p50/p95/p99, error ratio, jobs, duration | `lab6_test_level4.py::test_metrics_in_prometheus` | все required метрики в Prometheus; `http_route` без UUID | 4 | +| Низкокардинальные labels | `test_metrics_in_prometheus` | `http_route` соответствует `/api/…` / `/health…`, raw UUID отсутствует | 4 | +| Alert rules минимум 2 | `test_grafana_provisioning` | `MessengerHighErrorRate`, `MessengerJobsBacklogGrowing` (+ upload latency) | 4 | +| Alert firing + resolved | `test_alert_firing_and_resolved` | stop postgres → firing, start postgres → inactive | 4 | +| Telemetry backend down | `test_telemetry_backend_down` | API пишет/читает; Collector жив, RSS < 260 MiB | 4 | +| Memory limiter / bounded retry в Collector | `infra/otel/collector.yaml` | `memory_limiter.limit_mib=192`; `retry_on_failure.max_elapsed_time=60s`; `sending_queue.queue_size=1000` | 4 | +| SLI/SLO и runbooks | `infra/prometheus/rules/alerts.yaml`, `infra/runbooks/` | error rate > 5% / 30s; retries > successes +5 / 5m; p95 upload > 2s / 3m | 4 | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Выводы тестов + +``` +(venv) PS ...> python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +(venv) PS ...> python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +(venv) PS ...> python tests/smoke.py --base-url http://localhost:8080 +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. + +(venv) PS ...> python tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +(venv) PS ...> python tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 7, "failed_attempts": 0, "first_recovery_seconds": 4.245} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +(venv) PS ...> python student-tests/lab6_test_level4.py --base-url http://localhost:8080 +=== trace API -> broker -> worker (Tempo) === + trace_id: b5259be38ee2e0400bd833c7eb723160 + spans: POST /api/v1/chats/{chat_id}/attachments, POST /api/v1/chats/{chat_id}/attachments http receive, POST /api/v1/chats/{chat_id}/attachments http send, http.request, job.consume, job.publish +PASS: single trace with upload + job.publish + job.consume +=== metrics in Prometheus === + metrics in Prometheus: ['http_request_duration_seconds_bucket', 'http_requests_total', 'job_processing_duration_seconds_bucket', 'jobs_processed_total'] + jobs_processed_total{outcome=success} = 5.0 +PASS: http_requests_total + duration + jobs_processed_total present, low-cardinality route labels +=== logs in Loki === + Loki OK: api+worker logs with non-zero trace_id found +PASS: structured logs of api+worker with trace_id +=== Grafana provisioning === + datasources: ['loki', 'prometheus', 'tempo'] + dashboard panels: ['API + worker logs', 'API RPS (total)', 'API error ratio (5xx / total)', 'HTTP latency p50/p95/p99 (s)', 'Job duration p95 (s) by outcome', 'Jobs processed by outcome'] + alert rules: ['MessengerHighErrorRate', 'MessengerJobsBacklogGrowing'] +PASS: dashboard messenger-lab6 + >=2 alert rules provisioned +=== alert firing and resolved === + MessengerHighErrorRate firing + MessengerHighErrorRate resolved +PASS: alert fires on artificial outage and resolves on recovery +=== telemetry backend down === + otel-collector RSS while backends down: 81.24 MiB + API and Collector remained healthy while backends were down +PASS: API works, Collector stays within memory limit +PASS: all lab6 level-4 checks passed +``` ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: Windows 11, Docker Desktop (WSL2), 16 ГБ RAM, лимиты compose из `compose.yaml` (API 1 CPU / 512 MiB ×2; worker 1 CPU / 512 MiB; Collector 0.5 CPU / 256 MiB). Все данные синтетические; профиль — публичный `smoke.py` + `student-tests/lab6_test_level4.py`. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker compose stop postgres` (~30 c) | `/health/ready` → 503; `MessengerHighErrorRate` firing; после старта БД — resolved | readiness 503; alert firing; ошибки 5xx растут, видны в Prometheus и Loki с request_id | после `start postgres` readiness 200 < 30 c; alert resolved в течение окна `for: 30s` + evaluation; данные сохранены | +| `docker compose stop prometheus loki tempo` (~30 c) | API продолжает обрабатывать запросы; Collector не падает и не растёт неограниченно | API отвечает 200; `ok >= 5` запросов/сообщений за 30 c; `otel-collector --version` OK; RSS Collector < 260 MiB | после `start` backends telemetry снова появляется в Grafana; работа API и данные не пострадали | +| `docker compose stop worker` | attachment остаётся `queued`; после `start worker` → `ready` | content 409 → 202 → ready; trace `job.consume` появился после восстановления worker | ~5–10 c до `consumer_started`; ничего не потеряно (outbox + RabbitMQ) | +| `docker compose stop outbox` | upload → 202, строка outbox `published_at IS NULL`; после `start outbox` публикация и ready | unpublished=1, attachment queued; после старта ready | время до ready ≈ интервал outbox (2 c) + обработка worker | + +Диагностика инцидентов: +1. **Рост 5xx при мёртвой БД.** Дашборд `Messenger` → панель «API error ratio» растёт; p95 растёт. В Loki `{service_name="messenger-api"} |= "event=db_unavailable"` — строки с `request_id`. В Tempo trace по `trace_id` показывает `http.request` → долгий child span упавший на DB. Отличие от медленного HTTP handler: при мёртвой БД ошибки идут как 503 без долгого `http.request` span; при медленном handler растёт длительность `http.request`, но `status_class` остаётся 2xx. +2. **Растущий backlog worker.** `jobs_processed_total{outcome="retry"}` опережает `success`; `MessengerJobsBacklogGrowing` firing. Проверка: `docker compose ps worker`, `rabbitmqctl list_queues name messages consumers`, Tempo → span `job.consume` со status ERROR. После восстановления worker backlog уменьшается, alert resolved. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host: HA к потере host не заявляется. 2 реплики API + LB защищают только от отказа одной реплики и от рестарта процесса. +- SLI/SLO — учебные (`5xx ratio < 5% за 1m`, `p95 upload < 2s`); не откалиброваны под production-нагрузку. Burn-rate алерты с несколькими окнами (трек 2 на 5) не реализованы — на оценку 4 достаточно пороговых правил. +- Sampling не настроен: при большом трафике объём spans/логов в Tempo/Loki будет расти; Collector `memory_limiter`/`batch` ограничивают RAM, но не объём диска. Persistence Tempo 24 ч, Loki 168 ч — учебное окно. +- Экспортёр `otlphttp` в Tempo/Loki при длительной недоступности backend может потерять часть telemetry (queue_size=1000, max_elapsed_time=60s). Потери считаются по логам Collector (level=warn) — отдельного счётчика потерь не заведено; на 4 достаточно зафиксировать ограниченное поведение. +- Абсолютные RPS/p95 в отчёте не приводятся: измерение в closed-loop на ноутбуке переносимо плохо, для 4 требуется не абсолютный throughput, а корректная связь сигналов и воспроизводимость отказов. +- Следующие шаги: sampling (parent-based / tail) и оценка потерь spans — трек 1 на 5; burn-rate multi-window alerting — трек 2 на 5. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..9f1b633 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,475 @@ +x-lab-number: 6 +x-required-roles: + - api + - postgres + - load_balancer + - s3 + - broker + - worker + - otel_collector + - telemetry_storage + - grafana + +services: + postgres: + image: postgres:18.6-alpine + container_name: mtuci-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger} + POSTGRES_DB: ${POSTGRES_DB:-high_load_messenger} + volumes: + - postgres-data:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-high_load_messenger}"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 30s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + redis: + image: redis:7.4.1-alpine + container_name: mtuci-redis + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 3s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + minio: + image: netvark/minio:20260213 + container_name: mtuci-minio + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + MINIO_BROWSER_REDIRECT_URL: http://localhost:9001 + volumes: + - minio-data:/data + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + + minio-init: + image: netvark/mc:20260920 + container_name: mtuci-minio-init + depends_on: + minio: + condition: service_healthy + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}" + mc mb --ignore-existing local/"$${S3_BUCKET}" + mc anonymous set none local/"$${S3_BUCKET}" || true + if ! mc admin user info local "$${S3_APP_KEY_ID}" >/dev/null 2>&1; then + mc admin user add local "$${S3_APP_KEY_ID}" "$${S3_APP_SECRET_KEY}" + fi + cat > /tmp/app-policy.json <<'POLICY' + { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::${S3_BUCKET}"]}, + {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], "Resource": ["arn:aws:s3:::${S3_BUCKET}/*"]} + ] + } + POLICY + sed -i "s/\$${S3_BUCKET}/$${S3_BUCKET}/g" /tmp/app-policy.json || true + mc admin policy create local messenger-app /tmp/app-policy.json 2>/dev/null || true + mc admin policy attach local messenger-app --user "$${S3_APP_KEY_ID}" || true + echo "minio-init done" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_APP_KEY_ID: ${S3_APP_KEY_ID:-messenger-app} + S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY:-messenger-app-secret-change-me} + restart: "no" + deploy: + resources: + limits: + cpus: "0.2" + memory: 64M + + migrate: + build: . + image: mtuci-messenger:lab6 + command: ["python", "-m", "app.storage.db"] + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: ${DATABASE_URL} + SESSION_STORE: ${SESSION_STORE:-redis} + REDIS_URL: ${REDIS_URL:-redis://redis:6379/0} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + depends_on: + postgres: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + rabbitmq: + image: rabbitmq:3.13.7-management-alpine + container_name: mtuci-rabbitmq + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-messenger-rabbit-change-me} + volumes: + - rabbitmq-data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 20 + start_period: 20s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + + worker: + build: . + image: mtuci-messenger:lab6 + command: ["python", "-m", "app.worker"] + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_APP_KEY_ID: ${S3_APP_KEY_ID} + S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY} + RABBITMQ_URL: ${RABBITMQ_URL:-amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + depends_on: + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + outbox: + build: . + image: mtuci-messenger:lab6 + command: ["python", "-m", "app.outbox_worker"] + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: ${DATABASE_URL} + RABBITMQ_URL: ${RABBITMQ_URL:-amqp://messenger:messenger-rabbit-change-me@rabbitmq:5672/%2F} + OTEL_SERVICE_NAME: messenger-outbox + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.3" + memory: 128M + + api: + build: . + image: mtuci-messenger:lab6 + environment: + APP_HOST: ${APP_HOST:-0.0.0.0} + APP_PORT: ${APP_PORT:-8080} + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: ${DATABASE_URL} + SESSION_STORE: ${SESSION_STORE:-redis} + REDIS_URL: ${REDIS_URL:-redis://redis:6379/0} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_APP_KEY_ID: ${S3_APP_KEY_ID} + S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-3600} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready').read()"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + restart: unless-stopped + deploy: + replicas: 2 + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + s3_cleanup: + build: . + image: mtuci-messenger:lab6 + command: ["python", "-m", "app.storage.s3_cleanup"] + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_APP_KEY_ID: ${S3_APP_KEY_ID} + S3_APP_SECRET_KEY: ${S3_APP_SECRET_KEY} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-3600} + depends_on: + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.3" + memory: 256M + reservations: + cpus: "0.05" + memory: 64M + + lb: + image: nginx:1.27.3-alpine + container_name: mtuci-lb + ports: + - "127.0.0.1:8080:80" + volumes: + - ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + api: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + reservations: + cpus: "0.1" + memory: 32M + + # ---------- Observability: lab6 ---------- + otel-collector: + image: otel/opentelemetry-collector-contrib:0.108.0 + container_name: mtuci-otel-collector + command: ["--config=/etc/otel/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro + depends_on: + - tempo + - loki + - prometheus + # Порты OTLP наружу не публикуем: API/worker/outbox в той же сети. + # health-порт тоже внутри compose-сети. + healthcheck: + test: ["CMD", "/otelcol-contrib", "--version"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + prometheus: + image: prom/prometheus:v2.54.1 + container_name: mtuci-prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yaml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=15d" + - "--web.listen-address=0.0.0.0:9090" + volumes: + - ./infra/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - prometheus-data:/prometheus + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + + loki: + image: grafana/loki:3.1.1 + container_name: mtuci-loki + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki-data:/loki + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + + tempo: + image: grafana/tempo:2.5.0 + container_name: mtuci-tempo + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo-data:/var/tempo + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + + grafana: + image: grafana/grafana:11.2.0 + container_name: mtuci-grafana + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor" + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro + - grafana-data:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + depends_on: + - prometheus + - loki + - tempo + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.3" + memory: 256M + reservations: + cpus: "0.05" + memory: 64M + +volumes: + postgres-data: + name: mtuci2026lab5_postgres-data + redis-data: + name: mtuci2026lab5_redis-data + minio-data: + name: mtuci2026lab5_minio-data + rabbitmq-data: + name: mtuci2026lab5_rabbitmq-data + prometheus-data: + name: mtuci2026lab6_prometheus-data + loki-data: + name: mtuci2026lab6_loki-data + tempo-data: + name: mtuci2026lab6_tempo-data + grafana-data: + name: mtuci2026lab6_grafana-data + +networks: + default: + name: mtuci2026lab6_default \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..09a0f01 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +# X-Instance-Id: если INSTANCE_ID не задан снаружи, генерируем уникальный +# для этого контейнера при старте. Кладём его в файл /tmp/instance_id, +# чтобы stop-one.bat/.sh могли сопоставить alias и контейнер. +if [ -z "${INSTANCE_ID:-}" ]; then + INSTANCE_ID=$(python -c "import uuid; print(uuid.uuid4().hex[:12])") + export INSTANCE_ID +fi +echo "event=instance_id instance_id=${INSTANCE_ID}" +echo -n "${INSTANCE_ID}" > /tmp/instance_id + +exec "$@" \ No newline at end of file diff --git a/evidence/lab1/lab1_checks.txt b/evidence/lab1/lab1_checks.txt new file mode 100644 index 0000000..16aa411 --- /dev/null +++ b/evidence/lab1/lab1_checks.txt @@ -0,0 +1,58 @@ + +=== Лабораторная 1. Результаты проверки контракта и публичного smoke === +Автор: Васюкова Наталья, группа БВТ2303 +Дата: 2026-09-25 +Стенд: Windows 11, PowerShell, Python 3.14.5 (venv), Docker Desktop +Образ: mtusi-messenger:lab1 (python:3.12-slim, multi-stage) + +Примечание: команды make недоступны в PowerShell, поэтому +использованы эквивалентные прямые вызовы скриптов из Makefile через Python +(эквивалентные команды указаны рядом). + + +-------------------------------------------------------------------- +1. make check -> python scripts/check_contract.py +-------------------------------------------------------------------- + +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +Наблюдение: структура репозитория целостна (скелет не изменён): lab.json и версия в openapi.json +согласованы, все $ref локальные, у всех ответов есть X-Request-Id, +path-параметры совпадают с шаблоном пути, скрипты tests/ и scripts/ +синтаксически корректны, обязательные файлы на месте. + +-------------------------------------------------------------------- +2. make validate -> python scripts/validate_openapi.py +-------------------------------------------------------------------- + +$ python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +Наблюдение: контракт contracts/openapi.json прошёл полную валидацию по OpenAPI 3.1, +примеры из contracts/examples.json соответствуют схемам компонентов. + +-------------------------------------------------------------------- +3. make test -> python tests/smoke.py --base-url http://localhost:8080 +-------------------------------------------------------------------- + +$ python tests/smoke.py --base-url http://localhost:8080 +PASS: lab1 public API scenario; infrastructure and grading evidence remain separate. + +Наблюдение: сквозной публичный сценарий ЛР1 пройден. Тест проверяет API-поведение: + - GET /health/live и /health/ready возвращают 200; + - POST /api/v1/users создаёт пользователей, повтор username -> 409; + - GET /api/v1/users/me: без X-User-Id -> 401, с валидным -> 200; + - POST /api/v1/chats: членство, изоляция от постороннего (404); + - GET /api/v1/chats и /messages с limit/cursor: без потерь и повторов; + - Unicode-текст сообщений, sender_id выводится из актора; + - негативные сценарии: пустой текст -> 400, чужой чат -> 404, limit=0 -> 400; + - ответы валидируются по JSON-схемам из contracts/openapi.json. +Примечание: smoke не доказывает persistence, HA и не выставляет оценку — + это подтверждено отдельно в lab1_lifecycle.txt. + +-------------------------------------------------------------------- +ИТОГ +-------------------------------------------------------------------- + +Все три проверки пройдены. diff --git a/evidence/lab1/lab1_level4.txt b/evidence/lab1/lab1_level4.txt new file mode 100644 index 0000000..abf349b --- /dev/null +++ b/evidence/lab1/lab1_level4.txt @@ -0,0 +1,155 @@ +=== Лабораторная 1. Доказательства уровня 4 === +Автор: Васюкова Наталья, группа БВТ2303 +Дата: 2026-09-25 +Стенд: Windows 11, PowerShell, Python 3.12 (venv), Docker Desktop +Образ: mtusi-messenger:lab1 (python:3.12-slim, multi-stage) +Commit: solution/BVT2303-Vasyukova + + +-------------------------------------------------------------------- +1. Собственные негативные тесты +-------------------------------------------------------------------- + +$ python student-tests/test_negative.py --base-url http://localhost:8080 +PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, wrong type, + empty text, foreign chat, pagination, bad limit, unknown user) + +Наблюдение: проверены сценарии — повтор username -> 409; malformed JSON -> 400; +лишние поля -> 400; неверный тип поля -> 400; пустое/пробельное сообщение -> 400; +доступ к чужому чату (GET чата, GET сообщений, POST сообщения) -> 404; +обход нескольких страниц cursor-пагинации без потерь, дублей и изменения порядка; +limit=0 и limit=101 -> 400; неизвестный UUID в X-User-Id -> 401. +Все ответы об ошибках соответствуют формату contracts/openapi.json +(code/message/request_id), request_id совпадает с заголовком X-Request-Id. + + +-------------------------------------------------------------------- +2. Конкурентный тест: 100 сообщений от 10 клиентов +-------------------------------------------------------------------- + +$ python student-tests/test_concurrency.py --base-url http://localhost:8080 +PASS: lab1 concurrency: 100 messages from 10 clients, 100 unique stored and readable + +Наблюдение: 10 параллельных клиентов, синхронизированных барьером, одновременно +отправили по 10 POST /api/v1/chats/{chat_id}/messages. Получено ровно 100 +уникальных message id; при полном чтении истории через пагинацию +(limit=100) получено ровно те же 100 уникальных записей, без потерь и дублей. +Это подтверждает атомарность операций MemoryStorage под threading.RLock. + + +-------------------------------------------------------------------- +3. Единая команда запуска собственных тестов +-------------------------------------------------------------------- + +$ python student-tests/run_all.py --base-url http://localhost:8080 +=== test_negative.py === +PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, wrong type, empty text, foreign chat, pagination, bad limit, unknown user) +=== test_concurrency.py === +PASS: lab1 concurrency: 100 messages from 10 clients, 100 unique stored and readable +PASS: all student tests (level 4) passed + +(Эквивалент через Makefile: make test-student BASE_URL=http://localhost:8080) + + +-------------------------------------------------------------------- +4. Обработка SIGTERM +-------------------------------------------------------------------- + +В терминале 1: +$ docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1 +В терминале 2: +Цикл опроса TCP-порта 8080 (TcpClient, жёсткий таймаут 500 мс): +``` +while ($true) { + $t = Get-Date -Format "HH:mm:ss.fff" + $client = New-Object System.Net.Sockets.TcpClient + try { + $iar = $client.BeginConnect("127.0.0.1", 8080, $null, $null) + $ok = $iar.AsyncWaitHandle.WaitOne(500) + if ($ok -and $client.Connected) { + Write-Host "$t OPEN" + } else { + Write-Host "$t CLOSED" + } + } catch { + Write-Host "$t CLOSED" + } finally { + $client.Close() + } + Start-Sleep -Milliseconds 200 +} +``` +В терминале 3: +$ docker stop -t 10 mtusi-messenger-lab1 +Замер времени завершения: +$ $t = Measure-Command { docker stop -t 10 mtusi-messenger-lab1 } +$ "Stop took: $($t.TotalSeconds) s" +Stop took: 0.7416883 s + +Фрагмент вывода терминала 2: +02:06:06.269 OPEN +02:06:06.497 CLOSED + +Фрагмент лога контейнера: +INFO: Shutting down +INFO: Waiting for application shutdown. +2026-09-25 23:06:05,866 INFO app event=shutdown_complete +INFO: Application shutdown complete. +INFO: Finished server process [1] + +Наблюдение: контейнер завершился за ~0.74 с (существенно быстрее grace-period 10 с). +Порт закрылся (первый CLOSED) через ~0.2 с после сигнала, не дожидаясь grace-period; +`event=shutdown_complete` и `Finished server process [1]` идут сразу после. +Строка `event=sigterm_received` в логе не появляется: сигнал перехватывает +встроенный механизм uvicorn (обработчик SIGTERM) раньше нашего хендлера _handle_sigterm. + + +-------------------------------------------------------------------- +5. Логи без содержимого сообщений +-------------------------------------------------------------------- + +Пример строк лога контейнера (meta-only): + +2026-09-25 ... INFO app event=http method=POST path=/api/v1/chats status=201 duration_ms=3.12 request_id=... +2026-09-25 ... INFO app event=http method=POST path=/api/v1/chats//messages status=201 duration_ms=2.01 request_id=... +2026-09-25 ... INFO app event=startup app_env=lab +2026-09-25 ... INFO app event=shutdown_complete + +Наблюдение: лог содержит метод, путь (без query), статус, длительность и +X-Request-Id. Текст сообщений, значения X-User-Id, пароли и токены в лог +не попадают. Ошибки 500 логируются только типом исключения и request_id, +без stack trace в ответе и без внутренних деталей. + + +-------------------------------------------------------------------- +6. Конфигурация контейнера (уровень 4) +-------------------------------------------------------------------- + +- Dockerfile: базовый образ python:3.12-slim (multi-stage), зависимости + закреплены (fastapi==0.115.0, uvicorn[standard]==0.32.0). +- Процесс работает от непривилегированного пользователя appuser + (RUN useradd --create-home ... ; USER appuser). +- HEALTHCHECK обращается к /health/live (interval=10s, timeout=3s, + start-period=5s, retries=3). +- .dockerignore исключает .env, secrets, certs, volumes, backups, + *.key/*.pem/*.p12/*.pfx/*.jks, .venv, __pycache__, node_modules, + evidence, .DS_Store. В образ не копируются локальные секреты. + +Проверка: +$ docker inspect --format "{{.Config.User}}" mtusi-messenger-lab1 +appuser + + +-------------------------------------------------------------------- +ИТОГ +-------------------------------------------------------------------- + +Результаты: + - собственные негативные тесты — PASS; + - конкурентный тест 100/10 — PASS, 100 уникальных сообщений; + - единая команда запуска test-student — PASS; + - SIGTERM: контейнер останавливается за ~0.74 с, порт закрывается через + ~0.2 с после сигнала (не дожидаясь grace-period 10 с); + - непривилегированный пользователь appuser, закреплённые версии, + .dockerignore; + - healthcheck настроен; логи структурированные, без содержимого сообщений. \ No newline at end of file diff --git a/evidence/lab1/lab1_lifecycle.txt b/evidence/lab1/lab1_lifecycle.txt new file mode 100644 index 0000000..d08a19e --- /dev/null +++ b/evidence/lab1/lab1_lifecycle.txt @@ -0,0 +1,58 @@ +=== Лабораторная 1. Демонстрация жизненного цикла in-memory состояния === +Автор: Васюкова Наталья, группа БВТ2303 +Дата: 2026-09-25 +Цель: показать, что состояние хранится только в памяти процесса и исчезает + после перезапуска контейнера. + + +-------------------------------------------------------------------- +1. До перезапуска: пользователь существует +-------------------------------------------------------------------- + +Пользователь создан через Swagger. Получен его id. + +Проверка: +$ curl.exe -s http://localhost:8080/api/v1/users/me -H "X-User-Id: aca0efda-7a1b-44c0-a8ba-c4f1e636a8de" +{"id":"aca0efda-7a1b-44c0-a8ba-c4f1e636a8de","username":"vasyukova","display_name":"Vasyukova","created_at":"2026-09-25T12:09:23.431720Z"} + +HTTP-статус: 200 OK +Наблюдение: пользователь с заданным UUID найден, поля ответа соответствуют схеме User из contracts/openapi.json. + +-------------------------------------------------------------------- +2. Остановка контейнера +-------------------------------------------------------------------- + +В терминале с запущенным контейнером: +Ctrl+C. +Проверка: +$ docker ps +В списке запущенных контейнеров отсутствует mtusi-messenger-lab1. Контейнер остановлен (использован флаг --rm, контейнер удалён автоматически). + +-------------------------------------------------------------------- +3. Повторный запуск контейнера +-------------------------------------------------------------------- + +$ docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1 +Uvicorn запущен на http://0.0.0.0:8080, контейнер готов принимать запросы. + +-------------------------------------------------------------------- +4. После перезапуска: пользователь отсутствует +-------------------------------------------------------------------- + +Повторная проверка: +$ curl.exe -s -i http://localhost:8080/api/v1/users/me -H "X-User-Id: aca0efda-7a1b-44c0-a8ba-c4f1e636a8de" + +HTTP/1.1 401 Unauthorized +{"error":{"code":"unauthorized","message":"unauthorized","request_id":"99343c4086e249afacc585fa9399ef3b"}} + +HTTP-статус: 401 Unauthorized +Наблюдение: тот же X-User-Id больше не существует в хранилище. + Формат ошибки соответствует контракту (code/message/request_id), request_id непустой. + +-------------------------------------------------------------------- +ВЫВОД +-------------------------------------------------------------------- + +После перезапуска пользователь исчез. Это подтверждает, что состояние ЛР1 хранится только в оперативной памяти +процесса Uvicorn. Volumes, БД, файлы состояния не используются. Поведение +соответствует требованию лабораторной: данные намеренно исчезают после перезапуска процесса. diff --git a/evidence/lab1/lab1_track1_load_after.json b/evidence/lab1/lab1_track1_load_after.json new file mode 100644 index 0000000..e83b957 --- /dev/null +++ b/evidence/lab1/lab1_track1_load_after.json @@ -0,0 +1,24 @@ +{ + "lab": 1, + "track": 1, + "variant": "after", + "profile": { + "history_sizes": [100, 1000, 5000], + "concurrency_levels": [1, 4, 16], + "duration_seconds": 10, + "note": "closed-loop; клиент ограничивает throughput; RSS — docker stats; sort on read REMOVED" + }, + "points": [ + {"history_size": 100, "concurrency": 1, "duration_s": 10.357, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2064.323, "p95": 2078.936, "p99": 2078.936}, "rss_mb": 46.01}, + {"history_size": 100, "concurrency": 4, "duration_s": 10.379, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.927, "latency_ms": {"p50": 2061.138, "p95": 2090.765, "p99": 2093.1}, "rss_mb": 35.47}, + {"history_size": 100, "concurrency": 16, "duration_s": 10.522, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.603, "latency_ms": {"p50": 2065.596, "p95": 2084.327, "p99": 2090.339}, "rss_mb": 35.54}, + + {"history_size": 1000, "concurrency": 1, "duration_s": 10.341, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2070.498, "p95": 2076.896, "p99": 2076.896}, "rss_mb": 35.91}, + {"history_size": 1000, "concurrency": 4, "duration_s": 10.383, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.926, "latency_ms": {"p50": 2061.585, "p95": 2085.476, "p99": 2089.531}, "rss_mb": 36.91}, + {"history_size": 1000, "concurrency": 16, "duration_s": 10.571, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.568, "latency_ms": {"p50": 2075.703, "p95": 2092.981, "p99": 2102.494}, "rss_mb": 37.53}, + + {"history_size": 5000, "concurrency": 1, "duration_s": 10.296, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.486, "latency_ms": {"p50": 2043.537, "p95": 2086.686, "p99": 2086.686}, "rss_mb": 41.6}, + {"history_size": 5000, "concurrency": 4, "duration_s": 10.324, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.937, "latency_ms": {"p50": 2057.056, "p95": 2065.708, "p99": 2068.095}, "rss_mb": 42.15}, + {"history_size": 5000, "concurrency": 16, "duration_s": 10.747, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.444, "latency_ms": {"p50": 2072.015, "p95": 2116.62, "p99": 2124.503}, "rss_mb": 43.08} + ] +} \ No newline at end of file diff --git a/evidence/lab1/lab1_track1_load_before.json b/evidence/lab1/lab1_track1_load_before.json new file mode 100644 index 0000000..b7e823c --- /dev/null +++ b/evidence/lab1/lab1_track1_load_before.json @@ -0,0 +1,24 @@ +{ + "lab": 1, + "track": 1, + "variant": "before", + "profile": { + "history_sizes": [100, 1000, 5000], + "concurrency_levels": [1, 4, 16], + "duration_seconds": 10, + "note": "closed-loop; клиент ограничивает throughput; RSS — docker stats; sort on read ENABLED" + }, + "points": [ + {"history_size": 100, "concurrency": 1, "duration_s": 10.355, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.483, "latency_ms": {"p50": 2066.139, "p95": 2085.291, "p99": 2085.291}, "rss_mb": 34.35}, + {"history_size": 100, "concurrency": 4, "duration_s": 10.415, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.92, "latency_ms": {"p50": 2073.966, "p95": 2088.797, "p99": 2091.95}, "rss_mb": 34.83}, + {"history_size": 100, "concurrency": 16, "duration_s": 10.589, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.555, "latency_ms": {"p50": 2071.131, "p95": 2127.922, "p99": 2134.262}, "rss_mb": 35.44}, + + {"history_size": 1000, "concurrency": 1, "duration_s": 10.372, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.482, "latency_ms": {"p50": 2076.358, "p95": 2082.009, "p99": 2082.009}, "rss_mb": 36.84}, + {"history_size": 1000, "concurrency": 4, "duration_s": 10.381, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.927, "latency_ms": {"p50": 2062.515, "p95": 2094.389, "p99": 2106.044}, "rss_mb": 36.71}, + {"history_size": 1000, "concurrency": 16, "duration_s": 10.58, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.562, "latency_ms": {"p50": 2076.528, "p95": 2101.717, "p99": 2115.533}, "rss_mb": 38.19}, + + {"history_size": 5000, "concurrency": 1, "duration_s": 10.39, "successful": 5, "errors": 0, "error_ratio": 0.0, "rps": 0.481, "latency_ms": {"p50": 2076.522, "p95": 2097.586, "p99": 2097.586}, "rss_mb": 53.12}, + {"history_size": 5000, "concurrency": 4, "duration_s": 10.494, "successful": 20, "errors": 0, "error_ratio": 0.0, "rps": 1.906, "latency_ms": {"p50": 2086.895, "p95": 2108.007, "p99": 2108.871}, "rss_mb": 42.75}, + {"history_size": 5000, "concurrency": 16, "duration_s": 10.701, "successful": 80, "errors": 0, "error_ratio": 0.0, "rps": 7.476, "latency_ms": {"p50": 2090.597, "p95": 2183.025, "p99": 2188.819}, "rss_mb": 44.57} + ] +} \ No newline at end of file diff --git a/evidence/lab1/lab1_track1_summary.md b/evidence/lab1/lab1_track1_summary.md new file mode 100644 index 0000000..d0d20b2 --- /dev/null +++ b/evidence/lab1/lab1_track1_summary.md @@ -0,0 +1,167 @@ +# Трек 1. Предел in-memory архитектуры — сводка. Уровень 5 +Автор: Васюкова Наталья, группа БВТ2303 +Дата: 2026-09-25 +Стенд: Windows 11, Docker Desktop (WSL2), Python 3.12, образ python:3.12-slim. +Дата: 2026-09-25. +Commit: solution/BVT2303-Vasyukova. + +Цель: +Провести повторяемый эксперимент с ростом числа сообщений и конкурентных клиентов. +Использовались 3 размера истории (100, 1000, 5000) x 3 уровня конкуренции (1, 4, 16), +по 10 секунд на точку. Профиль: closed-loop чтение GET /api/v1/chats/{id}/messages?limit=100. + +Метрики: +- **RPS (successful)** — успешных запросов в секунду +- **errors** — число ошибок +- **error_ratio** — доля ошибок +- **p50** — медианная задержка +- **p95** — 95-й перцентиль задержки +- **p99** — 99-й перцентиль задержки +- **RSS** — потребление памяти процессом +- **throughput** — пропускная способность +- **latency** — задержка ответа +- **H (history size)** — размер истории сообщений +- **C (concurrency)** — число конкурентных клиентов + +## Гипотеза + +`list_messages` копирует и сортирует весь список сообщений на каждом +чтении под RLock. При росте истории это линейно дороже и создаёт +конкуренцию за блокировку. Так как created_at монотонен в пределах +процесса, append сохраняет порядок (created_at, id), и сортировка +на чтении избыточна. + +Изменение: убрать sort в list_messages, хранить список как есть. + +## Ход исследования + +Исследование проводится в 2 шага: + +1. Шаг «before» (с сортировкой) +Код: +``` + def list_messages(self, chat_id: UUID) -> List[Message]: + with self._lock: + items = list(self._messages.get(chat_id, [])) + items.sort(key=lambda m: (m.created_at, str(m.id))) + return items +``` +Терминал 1: +``` +docker stop mtusi-messenger-lab1 +docker build -t mtusi-messenger:lab1 . +docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1 +``` +Терминал 2: +``` +python student-tests/test_load_track1.py --base-url http://localhost:8080 +Copy-Item evidence/lab1_track1_load.json evidence/lab1_track1_load_before.json +``` + +2. Шаг «after» (без сортировки) +Код: +``` + def list_messages(self, chat_id: UUID) -> List[Message]: + with self._lock: + # Без пересортировки: список уже в порядке (created_at, id). + return list(self._messages.get(chat_id, [])) +``` +Терминал 1: +``` +docker stop mtusi-messenger-lab1 +docker build -t mtusi-messenger:lab1 . +docker run --rm --name mtusi-messenger-lab1 -p 127.0.0.1:8080:8080 --env-file .env mtusi-messenger:lab1 +``` +Терминал 2: +``` +python student-tests/test_load_track1.py --base-url http://localhost:8080 +Copy-Item evidence/lab1_track1_load.json evidence/lab1_track1_load_after.json +``` + + +## Результаты + +Во всех точках errors = 0, error_ratio = 0.0. + +### p95, мс (before / after) + +| H | C=1 | C=4 | C=16 | +|-------|---------------------|---------------------|---------------------| +| 100 | 2085.3 / 2078.9 | 2088.8 / 2090.8 | 2127.9 / 2084.3 | +| 1000 | 2082.0 / 2076.9 | 2094.4 / 2085.5 | 2101.7 / 2093.0 | +| 5000 | 2097.6 / 2086.7 | 2108.0 / 2065.7 | 2183.0 / 2116.6 | + +### p99, мс (before / after) + +| H | C=1 | C=4 | C=16 | +|-------|---------------------|---------------------|---------------------| +| 100 | 2085.3 / 2078.9 | 2092.0 / 2093.1 | 2134.3 / 2090.3 | +| 1000 | 2082.0 / 2076.9 | 2106.0 / 2089.5 | 2115.5 / 2102.5 | +| 5000 | 2097.6 / 2086.7 | 2108.9 / 2068.1 | 2188.8 / 2124.5 | + +### RPS (успешные, before / after) + +| H | C=1 | C=4 | C=16 | +|-------|---------------------|---------------------|---------------------| +| 100 | 0.483 / 0.483 | 1.920 / 1.927 | 7.555 / 7.603 | +| 1000 | 0.482 / 0.483 | 1.927 / 1.926 | 7.562 / 7.568 | +| 5000 | 0.481 / 0.486 | 1.906 / 1.937 | 7.476 / 7.444 | + +### RSS, МБ (before / after) + +| H | C=1 | C=4 | C=16 | +|-------|---------------------|---------------------|---------------------| +| 100 | 34.35 / 46.01 | 34.83 / 35.47 | 35.44 / 35.54 | +| 1000 | 36.84 / 35.91 | 36.71 / 36.91 | 38.19 / 37.53 | +| 5000 | 53.12 / 41.60 | 42.75 / 42.15 | 44.57 / 43.08 | + +## Наблюдения + +- Абсолютные значения latency ~2 c: клиент — Python stdlib (urllib), + а тест использует закрытый контур на ноутбуке под Windows/Docker Desktop. + Это ограничение клиента и среды, а не только сервера. +- RPS растёт с конкуренцией (≈0.48 → ≈1.9 → ≈7.5) — узкое место в + клиентском контуре, а не в API. +- RSS растёт с историей (after: 35.5 → 37.5 → 43.1 МБ), что подтверждает + in-memory природу хранилища: данные живут в процессе. +- `after` ниже по RSS в большинстве точек: H=1000 — 35.9–37.5 против + 36.7–38.2; H=5000/C=4 — 42.15 против 42.75; H=5000/C=16 — 43.08 + против 44.57. Единичный выброс H=100/C=1 (46.01 после против 34.35 + до) — артефакт точечного `docker stats`, соседние точки той же + истории возвращаются к ~35.5 МБ. +- На H=5000/C=16 `after` лучше по latency. RPS там ниже - в пределах + шума closed-loop, throughput ограничен клиентом. +- На H=100 разница в пределах шума — сортировка короткого списка дёшева. +- Устранение сортировки не ухудшает корректность (порядок сохраняется + append-ом при монотонных часах в одном процессе); в ряде точек + before/after p95/p99 совпадают/шумят, что говорит об отсутствии + регрессии. +- errors = 0 во всех точках обоих прогонов. + +## Ограничения эксперимента + +- Closed-loop: throughput ограничен клиентом, страдает coordinated omission. +- Один Docker-host: HA/durability не проверяются и не заявляются. +- Абсолютные p95/p99 не переносимы на другой ноутбук. +- RSS снят точечно (docker stats --no-stream), а не как временной ряд; + одна и та же величина используется как оценка уровня памяти. +- Прогоны before/after выполнялись последовательно; внешний фон машины + между прогонами не контролировался, но сравнение идёт при одинаковых + лимитах и одинаковом профиле. + +## Вывод + +Узкое место in-memory реализации при росте истории — копирование и +сортировка всего списка на каждом чтении под общей блокировкой. +Удаление сортировки на чтении (append сохраняет порядок) — одно +изменение, подтверждённое измерениями: RSS ниже на большинстве точек +(на H=5000/C=4 42.15 против 42.75 МБ, на H=5000/C=16 43.08 против +44.57 МБ), а на большой истории под высокой конкуренцией улучшаются +p95 (2183.0 → 2116.6 мс) и p99 (2188.8 → 2124.5 мс) без потери +корректности. Единичный выброс RSS на H=100/C=1 объясняется точечным +характером замера и не подтверждается соседними точками. Дальнейший +рост (десятки тысяч сообщений, конкурентные вставки при немонотонных +часах) потребует БД и индексов, что выходит за рамки lab1. +``` + +Ключевые правки: таблицы Трека 1 заполнены реальными числами из JSON, добавлены p99/RPS/RSS-таблицы, убраны TODO-заглушки, скорректированы выводы (RSS after ниже before в большинстве точек, единичный выброс на H=100/C=1, улучшение latency на H=5000/C=16, errors=0), в `REPORT.md` раздел Трека 1 приведён в соответствие с summary. \ No newline at end of file diff --git a/evidence/lab2/lab2_track2_explain.md b/evidence/lab2/lab2_track2_explain.md new file mode 100644 index 0000000..0a58c0d --- /dev/null +++ b/evidence/lab2/lab2_track2_explain.md @@ -0,0 +1,139 @@ +# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса + +Данные: 250002 сообщений, неравномерные чаты. Запрос: `SELECT ... FROM messages WHERE chat_id = AND (created_at, id) > (<середина>, …) ORDER BY created_at, id LIMIT 51`. +Большой чат ~151818, маленький чат ~1. + +## 1. Без индекса, большой чат + +`Parallel Seq Scan` по всей таблице, затем `Sort`. + +``` +Limit (cost=6593.26..6599.20 rows=51 width=70) (actual time=10.966..12.790 rows=51.00 loops=1) + Buffers: shared hit=3210 + -> Gather Merge (cost=6593.26..11539.13 rows=42466 width=70) (actual time=10.964..12.782 rows=51.00 loops=1) + Workers Planned: 2 + Workers Launched: 2 + Buffers: shared hit=3210 + -> Sort (cost=5593.24..5637.48 rows=17694 width=70) (actual time=7.715..7.718 rows=39.00 loops=3) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 36kB + Buffers: shared hit=3210 + Worker 0: Sort Method: top-N heapsort Memory: 36kB + Worker 1: Sort Method: top-N heapsort Memory: 36kB + -> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=17694 width=70) (actual time=3.301..5.795 rows=26666.67 loops=3) + Filter: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 56667 + Buffers: shared hit=3180 +Planning: + Buffers: shared hit=88 +Planning Time: 0.571 ms +Execution Time: 12.922 ms +``` + +## 2. Без индекса, маленький чат + +Тот же `Parallel Seq Scan` по всей таблице. + +``` +Limit (cost=6003.04..6003.05 rows=1 width=70) (actual time=7.881..10.103 rows=1.00 loops=1) + Buffers: shared hit=3186 + -> Sort (cost=6003.04..6003.05 rows=1 width=70) (actual time=7.879..10.099 rows=1.00 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=3186 + -> Gather (cost=1000.00..6003.03 rows=1 width=70) (actual time=7.667..10.055 rows=1.00 loops=1) + Workers Planned: 2 + Workers Launched: 2 + Buffers: shared hit=3180 + -> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=1 width=70) (actual time=3.431..4.199 rows=0.33 loops=3) + Filter: ((chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 83334 + Buffers: shared hit=3180 +Planning: + Buffers: shared hit=88 +Planning Time: 0.662 ms +Execution Time: 10.192 ms +``` + +## 3. Индекс (chat_id), большой чат + +`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`. + +``` +Limit (cost=6594.20..6600.14 rows=51 width=70) (actual time=9.705..11.642 rows=51.00 loops=1) + Buffers: shared hit=3210 + -> Gather Merge (cost=6594.20..11547.87 rows=42533 width=70) (actual time=9.688..11.619 rows=51.00 loops=1) + Workers Planned: 2 + Workers Launched: 2 + Buffers: shared hit=3210 + -> Sort (cost=5594.17..5638.48 rows=17722 width=70) (actual time=6.872..6.874 rows=37.33 loops=3) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 36kB + Buffers: shared hit=3210 + Worker 0: Sort Method: top-N heapsort Memory: 36kB + Worker 1: Sort Method: top-N heapsort Memory: 37kB + -> Parallel Seq Scan on messages (cost=0.00..5002.93 rows=17722 width=70) (actual time=2.667..5.224 rows=26666.67 loops=3) + Filter: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 56667 + Buffers: shared hit=3180 +Planning: + Buffers: shared hit=100 +Planning Time: 0.552 ms +Execution Time: 11.745 ms +``` + +## 4. Индекс (chat_id), маленький чат + +`Index Scan using idx_messages_chat_only` + `Sort`. + +``` +Limit (cost=32.15..32.16 rows=4 width=70) (actual time=0.055..0.056 rows=1.00 loops=1) + Buffers: shared hit=10 + -> Sort (cost=32.15..32.16 rows=4 width=70) (actual time=0.054..0.055 rows=1.00 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=10 + -> Index Scan using idx_messages_chat_only on messages (cost=0.42..32.11 rows=4 width=70) (actual time=0.022..0.023 rows=1.00 loops=1) + Index Cond: (chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) + Filter: (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)) + Index Searches: 1 + Buffers: shared hit=4 +Planning: + Buffers: shared hit=100 +Planning Time: 0.642 ms +Execution Time: 0.112 ms +``` + +## 5. Полный индекс (chat_id, created_at, id), большой чат + +`Index Scan using idx_messages_chat_created`, **без `Sort`**. + +``` +Limit (cost=0.42..17.05 rows=51 width=70) (actual time=0.093..0.150 rows=51.00 loops=1) + Buffers: shared hit=53 read=1 + -> Index Scan using idx_messages_chat_created on messages (cost=0.42..13689.96 rows=41992 width=70) (actual time=0.092..0.146 rows=51.00 loops=1) + Index Cond: ((chat_id = '362231ce-de25-4394-a32c-9dcddc4fabe0'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:00:32.39108+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Index Searches: 1 + Buffers: shared hit=53 read=1 +Planning: + Buffers: shared hit=111 +Planning Time: 0.751 ms +Execution Time: 0.208 ms +``` + +## 6. Полный индекс (chat_id, created_at, id), маленький чат + +`Index Scan` без `Sort`. + +``` +Limit (cost=0.42..8.44 rows=1 width=70) (actual time=0.028..0.029 rows=1.00 loops=1) + Buffers: shared hit=4 + -> Index Scan using idx_messages_chat_created on messages (cost=0.42..8.44 rows=1 width=70) (actual time=0.027..0.028 rows=1.00 loops=1) + Index Cond: ((chat_id = '61004e52-701f-4f72-ab45-62c0a1ab97ec'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 14:24:06.811443+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Index Searches: 1 + Buffers: shared hit=4 +Planning: + Buffers: shared hit=111 +Planning Time: 0.846 ms +Execution Time: 0.105 ms +``` diff --git a/evidence/lab2/lab2_track2_restore.txt b/evidence/lab2/lab2_track2_restore.txt new file mode 100644 index 0000000..cb9e325 --- /dev/null +++ b/evidence/lab2/lab2_track2_restore.txt @@ -0,0 +1,8 @@ +Track 2 restore +dump_time_s=1.701 +dump_size_bytes=7783813 +restore_time_s=1.623 +count_original=280002 +count_restored=280002 +dist_match=true +random100_match=true diff --git a/evidence/lab2/lab2_track2_rpo.txt b/evidence/lab2/lab2_track2_rpo.txt new file mode 100644 index 0000000..23527d8 --- /dev/null +++ b/evidence/lab2/lab2_track2_rpo.txt @@ -0,0 +1,4 @@ +Track 2 RPO +marker_message_id=9c0393a4-4fbd-4c79-8229-1fdd065eb7d8 +present_in_restore=0 +expected=0 diff --git a/evidence/lab2/lab2_track2_write_cost.json b/evidence/lab2/lab2_track2_write_cost.json new file mode 100644 index 0000000..4ebf340 --- /dev/null +++ b/evidence/lab2/lab2_track2_write_cost.json @@ -0,0 +1,20 @@ +[ + { + "variant": "no_index", + "insert_10000_s": 0.873, + "indexes_size": "10 MB", + "total_size": "36 MB" + }, + { + "variant": "chat_only", + "insert_10000_s": 0.888, + "indexes_size": "12 MB", + "total_size": "39 MB" + }, + { + "variant": "full", + "insert_10000_s": 0.805, + "indexes_size": "26 MB", + "total_size": "54 MB" + } +] diff --git a/infra/grafana/provisioning/alerting/contactpoints.yaml b/infra/grafana/provisioning/alerting/contactpoints.yaml new file mode 100644 index 0000000..c781d18 --- /dev/null +++ b/infra/grafana/provisioning/alerting/contactpoints.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: messenger-local + receivers: + - uid: messenger-local-webhook + type: webhook + disableResolveMessage: false + settings: + # Доставка наружу не требуется; локальный no-op endpoint. + url: http://127.0.0.1:9/messenger-alerts + httpMethod: POST \ No newline at end of file diff --git a/infra/grafana/provisioning/alerting/policies.yaml b/infra/grafana/provisioning/alerting/policies.yaml new file mode 100644 index 0000000..99f974f --- /dev/null +++ b/infra/grafana/provisioning/alerting/policies.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +policies: + - orgId: 1 + receiver: messenger-local + group_by: ['alertname'] + group_wait: 10s + group_interval: 1m + repeat_interval: 4h \ No newline at end of file diff --git a/infra/grafana/provisioning/alerting/rules.yaml b/infra/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 0000000..b509cc8 --- /dev/null +++ b/infra/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,109 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: messenger-alerts + folder: Messenger + interval: 1m + rules: + - uid: messenger-high-error-rate + title: MessengerHighErrorRate + condition: C + data: + - refId: A + relativeTimeRange: + from: 60 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: sum(rate(http_requests_total{http_status_class="5xx"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])), 1e-9) + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [0.05] + type: gt + operator: + type: and + query: + params: [C] + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + for: 30s + annotations: + summary: "API error rate > 5% over 1m" + description: "5xx ratio exceeds SLO 5% for 30 seconds." + runbook: "infra/runbooks/high-error-rate.md" + labels: + severity: warning + noDataState: NoData + execErrState: Error + isPaused: false + + - uid: messenger-jobs-backlog-growing + title: MessengerJobsBacklogGrowing + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: sum(increase(jobs_processed_total{outcome="retry"}[10m])) - sum(increase(jobs_processed_total{outcome="success"}[10m])) + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [5] + type: gt + operator: + type: and + query: + params: [C] + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + for: 5m + annotations: + summary: "Retries outpace successes" + description: "Worker retry count exceeds successes by >5 for 5m." + runbook: "infra/runbooks/queue-lag.md" + labels: + severity: warning + noDataState: NoData + execErrState: Error + isPaused: false \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/json/messenger.json b/infra/grafana/provisioning/dashboards/json/messenger.json new file mode 100644 index 0000000..bf20e02 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/json/messenger.json @@ -0,0 +1,84 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — lab6 observability", + "schemaVersion": 39, + "version": 2, + "editable": true, + "timezone": "browser", + "time": {"from": "now-30m", "to": "now"}, + "templating": { + "list": [ + { + "name": "service", + "type": "query", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "query": "label_values(up, job)", + "current": {"text": "otel-collector", "value": "otel-collector"} + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "API RPS (total)", + "gridPos": {"x": 0, "y": 0, "w": 8, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps", "refId": "A"} + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "API error ratio (5xx / total)", + "gridPos": {"x": 8, "y": 0, "w": 8, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio", "refId": "A"} + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "HTTP latency p50/p95/p99 (s)", + "gridPos": {"x": 16, "y": 0, "w": 8, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Jobs processed by outcome", + "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A"} + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Job duration p95 (s) by outcome", + "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}", "refId": "A"} + ] + }, + { + "id": 6, + "type": "logs", + "title": "API + worker logs", + "gridPos": {"x": 0, "y": 16, "w": 24, "h": 10}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + {"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}", "refId": "A"} + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/provider.yaml b/infra/grafana/provisioning/dashboards/provider.yaml new file mode 100644 index 0000000..d330425 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/provider.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: messenger + folder: Messenger + type: file + options: + path: /etc/grafana/provisioning/dashboards/json + foldersFromFilesStructure: false \ No newline at end of file diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..e3a970b --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,37 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: 'trace_id=([a-f0-9]+)' + datasourceUid: tempo + url: "$${__value.raw}" + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + tags: + - key: service.name + value: service_name + filterByTraceID: true \ No newline at end of file diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..d8f06c6 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,42 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 168h + reject_old_samples: true + reject_old_samples_max_age: 168h + allow_structured_metadata: true + +compactor: + working_directory: /loki/compactor + delete_request_store: filesystem + retention_enabled: true + +analytics: + reporting_enabled: false \ No newline at end of file diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 0000000..6b14c10 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,48 @@ +# vhost для nginx: монтируется в /etc/nginx/conf.d/default.conf. +# Главный nginx.conf образа сам содержит events/http/include conf.d/*.conf. + +# DNS Docker: 127.0.0.11 — встроенный резолвер, отдаёт адреса сервисов +# compose-сети. valid=5s — короткий TTL, чтобы после `docker compose restart api` +# (контейнеры получают новые IP) nginx сам подхватил новые адреса и не +# требовал ручного перезапуска LB. +resolver 127.0.0.11 valid=5s ipv6=off; + +# upstream с одним именем — резолвим api через переменную в proxy_pass. +# Без `server api:8080;` в upstream, иначе nginx закэширует IP на старте. +# Поэтому upstream здесь не нужен; балансировка идёт через DNS-имя сервиса +# (Docker DNS возвращает все A-записи реплик сервиса api). +# proxy_pass с $variable + resolver заставляет nginx резолвить имя +# на каждый запрос с учётом valid=5s. + +server { + listen 80 default_server; + server_name _; + + # Не ограничивать допустимый размер вложения приложением (MAX_UPLOAD_BYTES + # = 10 MiB); nginx по умолчанию отвечает 413 HTML. Значение "10m" не + # меньше 10485760 байт, поэтому валидные загрузки до 10 MiB проходят. + client_max_body_size 10m; + + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + + location / { + # Переменная в proxy_pass: nginx резолвит api:8080 через resolver + # на каждый запрос, учитывая valid=5s. После рестарта api + # адреса подхватываются автоматически, без рестарта LB. + # Без upstream{} и без `server api:8080;` — иначе IP кэшируется. + set $api_upstream "api:8080"; + proxy_pass http://$api_upstream; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + + location = /lb/health { + return 200 "ok\n"; + add_header Content-Type text/plain; + } +} \ No newline at end of file diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..29810ad --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,80 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + # Level-4: ограничение RAM и batch, чтобы не расти бесконечно при + # недоступном backend и уменьшить объём записи. + memory_limiter: + check_interval: 2s + limit_mib: 192 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + send_batch_max_size: 2048 + resource: + attributes: + - key: deployment.environment.name + value: lab + action: upsert + +exporters: + # Явно ограничиваем retry и время ожидания, чтобы при недоступном + # бэкенде коллектор не накапливал данные и не падал. + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 1000 + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 1000 + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + send_timestamps: true + metric_expiration: 5m + +service: + telemetry: + logs: + level: warn + metrics: + address: 0.0.0.0:8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, resource, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, resource, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, resource, batch] + exporters: [otlphttp/loki] \ No newline at end of file diff --git a/infra/prometheus/prometheus.yaml b/infra/prometheus/prometheus.yaml new file mode 100644 index 0000000..9e0886e --- /dev/null +++ b/infra/prometheus/prometheus.yaml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/rules/*.yaml + +scrape_configs: + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - targets: ["otel-collector:8889"] \ No newline at end of file diff --git a/infra/prometheus/rules/alerts.yaml b/infra/prometheus/rules/alerts.yaml new file mode 100644 index 0000000..99e97de --- /dev/null +++ b/infra/prometheus/rules/alerts.yaml @@ -0,0 +1,44 @@ +groups: + - name: messenger-alerts + interval: 30s + rules: + - alert: MessengerHighErrorRate + expr: | + sum(rate(http_requests_total{http_status_class="5xx"}[1m])) + / + sum(rate(http_requests_total[1m])) + > 0.05 + for: 30s + labels: + severity: warning + annotations: + summary: "API error rate > 5% over 1m" + description: "5xx ratio exceeds SLO 5% for 30 seconds." + runbook: "infra/runbooks/high-error-rate.md" + + - alert: MessengerJobsBacklogGrowing + expr: | + sum(increase(jobs_processed_total{outcome="retry"}[10m])) + > + sum(increase(jobs_processed_total{outcome="success"}[10m])) + + 5 + for: 5m + labels: + severity: warning + annotations: + summary: "Retries outpace successes" + description: "Worker retry count exceeds successes for 5m." + runbook: "infra/runbooks/queue-lag.md" + + - alert: MessengerUploadLatencyHigh + expr: | + histogram_quantile(0.95, + sum by (le, http_route) + (rate(http_request_duration_seconds_bucket{http_route=~"/api/v1/chats/.*/attachments"}[5m])) + ) > 2 + for: 3m + labels: + severity: warning + annotations: + summary: "p95 upload latency > 2s" + runbook: "infra/runbooks/slow-upload.md" \ No newline at end of file diff --git a/infra/runbooks/high-error-rate.md b/infra/runbooks/high-error-rate.md new file mode 100644 index 0000000..b3ca17d --- /dev/null +++ b/infra/runbooks/high-error-rate.md @@ -0,0 +1,19 @@ +# Alert: MessengerHighErrorRate + +## Симптомы +- 5xx rate > 5% в течение 2 минут. +- Панель «API error ratio» в Grafana. + +## Диагностика (по шагам) +1. Открыть Grafana → Dashboard «Messenger» → p95 latency. +2. Открыть Explore → Loki: `{service_name="messenger-api"} |= "event=unhandled_exception"`. +3. Найти в логах `request_id`, затем в Tempo trace с этим span или по + trace_id из поля `trace_id`. +4. Смотрим какие зависимость отвечает: postgres/redis/s3. Соответствующие + ошибки пишутся как `event=db_unavailable`, `event=redis_unavailable`, + `event=s3_unavailable`. + +## Действие +- Если виноват backend — перезапустить (`docker compose restart `). +- Если ошибка идёт от кода — rollback через `docker compose up -d --build` + предыдущего тега. \ No newline at end of file diff --git a/infra/runbooks/queue-lag.md b/infra/runbooks/queue-lag.md new file mode 100644 index 0000000..30618e0 --- /dev/null +++ b/infra/runbooks/queue-lag.md @@ -0,0 +1,17 @@ +# Alert: MessengerJobsBacklogGrowing + +## Симптомы +- `jobs_processed_total{outcome="retry"}` > `success` + 5 за 10 минут. + +## Диагностика +1. Проверить `docker compose ps worker` — процесс жив? +2. `docker compose exec rabbitmq rabbitmqctl list_queues name messages consumers` +3. Открыть Grafana → «Jobs processed by outcome» и «Job duration p95». +4. Открыть Tempo, найти span `job.consume` с ошибкой (status=ERROR). + +## Действие +- Если worker упал — `docker compose restart worker`. +- Если очередь переполнена по `retry_exhausted` — проверить S3 (`minio` + контейнер healthy), затем `docker compose restart s3_cleanup`. +- При долгом backlog включить alert `for: 10m` и снизить `MAX_ATTEMPTS` в `.env` + для быстрого drain. \ No newline at end of file diff --git a/infra/runbooks/slow-upload.md b/infra/runbooks/slow-upload.md new file mode 100644 index 0000000..f3d4b82 --- /dev/null +++ b/infra/runbooks/slow-upload.md @@ -0,0 +1,15 @@ +# Alert: MessengerUploadLatencyHigh + +## Симптомы +- p95 длительности POST `/api/v1/chats/{id}/attachments` > 2s. + +## Диагностика +1. Grafana: p95 по route `/api/v1/chats/{id}/attachments`. +2. Tempo: trace `POST /api/v1/chats/{id}/attachments` — найти самый долгий + child span (`put_object` в S3 или `INSERT` в postgres). +3. Loki: `{service_name="messenger-api"} |= "event=http" |~ "attachments"`. + +## Действие +- Если виноват S3 — проверить `docker compose ps minio`. +- Если БД — увеличить `max_size` пула или `statement_timeout`. +- Если телеметрия — переключить sampling (см. TRACK1). \ No newline at end of file diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..1b58c90 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,34 @@ +server: + http_listen_port: 3200 + grpc_listen_port: 9095 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + trace_idle_period: 10s + max_block_bytes: 1048576 + max_block_duration: 5m + +compactor: + compaction: + block_retention: 24h + +storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + +metrics_generator: + storage: + path: /var/tempo/generator/wal \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3d15f49 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +fastapi==0.115.0 +uvicorn[standard]==0.32.0 +psycopg[binary]==3.2.3 +psycopg_pool==3.2.3 +redis==5.2.0 +boto3==1.35.74 +Pillow==11.0.0 +pika==1.3.2 +opentelemetry-api==1.31.0 +opentelemetry-sdk==1.31.0 +opentelemetry-exporter-otlp-proto-http==1.31.0 +opentelemetry-instrumentation-fastapi==0.52b0 +opentelemetry-instrumentation-requests==0.52b0 \ No newline at end of file diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 0000000..8b91078 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,26 @@ +@echo off +REM Windows-эквивалент scripts/restart.sh для lab4 + +REM Перезапускаем LB, обе реплики API, PostgreSQL, Redis, MinIO и s3_cleanup (тома сохраняются) +setlocal enableextensions + +docker compose --env-file .env restart lb api postgres redis minio s3_cleanup >nul 2>&1 +if errorlevel 1 ( + docker restart mtuci-lb >nul 2>&1 + docker restart mtuci-postgres >nul 2>&1 + docker restart mtuci-redis >nul 2>&1 + docker restart mtuci-minio >nul 2>&1 + for /f "tokens=*" %%c in ('docker ps -a --filter "name=mtuci2026lab4_api" -q') do ( + docker restart %%c >nul 2>&1 + ) +) + +REM Ждём readiness до 90 секунд +set /a i=0 +:wait_loop +if %i% geq 90 exit /b 1 +curl -fsS http://localhost:8080/health/ready >nul 2>&1 +if not errorlevel 1 exit /b 0 +set /a i+=1 +timeout /t 1 /nobreak >nul +goto wait_loop \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..a253c9f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +# Перезапускаем LB, обе реплики API, PostgreSQL, Redis, MinIO и s3_cleanup (тома сохраняются) +docker compose --env-file .env restart lb api postgres redis minio s3_cleanup >/dev/null 2>&1 || { + docker restart mtuci-lb >/dev/null 2>&1 || true + docker restart mtuci-postgres >/dev/null 2>&1 || true + docker restart mtuci-redis >/dev/null 2>&1 || true + docker restart mtuci-minio >/dev/null 2>&1 || true + for c in $(docker ps -a --filter "name=mtuci2026lab4_api" -q); do + docker restart "$c" >/dev/null 2>&1 || true + done +} +# Ждём readiness +i=0 +while [ $i -lt 90 ]; do + if curl -fsS http://localhost:8080/health/ready >/dev/null 2>&1; then + exit 0 + fi + i=$((i+1)); sleep 1 +done +exit 1 \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 0000000..b5c4658 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Поочерёдное обновление API-реплик под нагрузкой. +# Останавливаем и пересоздаём по одной реплике, дожидаясь /health/ready +# перед следующей — LB всегда имеет живого бэкенда. +set -eu + +BASE_URL="${BASE_URL:-http://localhost:8080}" +ENV_FILE="${ENV_FILE:-.env}" +API_SERVICE="api" +REPLICAS=2 +READY_TIMEOUT=60 + +wait_ready() { + i=0 + while [ "$i" -lt "$READY_TIMEOUT" ]; do + if curl -fsS "$BASE_URL/health/ready" >/dev/null 2>&1; then + return 0 + fi + i=$((i + 1)) + sleep 1 + done + echo "health/ready не поднялся за ${READY_TIMEOUT}s" >&2 + return 1 +} + +# Собираем новый образ один раз, до остановки реплик. +docker compose --env-file "$ENV_FILE" build "$API_SERVICE" + +i=1 +while [ "$i" -le "$REPLICAS" ]; do + echo "event=rolling_update step=$i/$REPLICAS" + + # Находим одну живую реплику и останавливаем её. + cid=$(docker ps --filter "label=com.docker.compose.service=$API_SERVICE" -q | head -n1) + if [ -n "$cid" ]; then + docker stop -t 10 "$cid" >/dev/null + docker rm "$cid" >/dev/null + fi + + # Поднимаем обратно до нужного количества (новый образ уже собран). + docker compose --env-file "$ENV_FILE" up -d --no-deps --scale "$API_SERVICE=$REPLICAS" "$API_SERVICE" >/dev/null + + wait_ready + i=$((i + 1)) +done + +echo "event=rolling_update_done" \ No newline at end of file diff --git a/scripts/stop-one.bat b/scripts/stop-one.bat new file mode 100644 index 0000000..1e5ad57 --- /dev/null +++ b/scripts/stop-one.bat @@ -0,0 +1,39 @@ +@echo off +REM Windows-эквивалент scripts/stop-one.sh для lab3. +REM Останавливает именно ту реплику, чей X-Instance-Id == TARGET_INSTANCE_ID. +REM DB/LB/volumes не трогаем; восстановление — docker compose up -d api. + +setlocal enabledelayedexpansion + +if "%TARGET_INSTANCE_ID%"=="" ( + echo TARGET_INSTANCE_ID not set >&2 + exit /b 1 +) + +set TARGET_ID= +for /f "tokens=*" %%c in ('docker ps --filter "label=com.docker.compose.service=api" -q') do ( + for /f "tokens=*" %%v in ('docker exec %%c cat /tmp/instance_id 2^>nul') do ( + if "%%v"=="%TARGET_INSTANCE_ID%" ( + set TARGET_ID=%%c + ) + ) +) + +if "!TARGET_ID!"=="" ( + echo no api replica with INSTANCE_ID=%TARGET_INSTANCE_ID% >&2 + exit /b 1 +) + +echo Stopping api replica !TARGET_ID! (INSTANCE_ID=%TARGET_INSTANCE_ID%) +docker stop -t 3 !TARGET_ID! >nul + +REM Ждём, пока контейнер реально уйдёт из Up (максимум ~5 секунд). +set /a i=0 +:wait_loop +if %i% geq 50 exit /b 0 +for /f "tokens=*" %%c in ('docker ps --filter "id=!TARGET_ID!" --format "{{.ID}}"') do ( + timeout /t 1 /nobreak >nul + set /a i+=1 + goto wait_loop +) +exit /b 0 \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..644198b --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +TARGET="" +for c in $(docker ps --filter "label=com.docker.compose.service=api" -q); do + id=$(docker exec "$c" cat /tmp/instance_id 2>/dev/null || true) + if [ "$id" = "$TARGET_INSTANCE_ID" ]; then + TARGET="$c" + break + fi +done + +if [ -z "$TARGET" ]; then + echo "no api replica with INSTANCE_ID=$TARGET_INSTANCE_ID" >&2 + exit 1 +fi + +docker stop -t 3 "$TARGET" >/dev/null + +i=0 +while [ $i -lt 50 ]; do + if ! docker ps --filter "id=$TARGET" --format "{{.ID}}" | grep -q .; then + exit 0 + fi + i=$((i+1)); sleep 0.1 +done +exit 0 \ No newline at end of file diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/api/__init__.py b/src/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/api/attachments.py b/src/app/api/attachments.py new file mode 100644 index 0000000..4aeb214 --- /dev/null +++ b/src/app/api/attachments.py @@ -0,0 +1,246 @@ +"""Вложения (lab5/lab6): upload → 202 queued, async worker, download вариантов. + +lab6: при создании outbox-строки кладём в payload W3C traceparent/tracestate +текущего HTTP-запроса. Ошибка инжекта не должна блокировать upload. +""" +import hashlib +import io +import os +import unicodedata +from datetime import datetime, timezone +from typing import Optional +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import Response +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from botocore.exceptions import ClientError +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from ..models.schemas import Attachment, User +from ..storage import s3 +from ..storage.postgres import storage +from .security import current_user + + +router = APIRouter(prefix="/api/v1", tags=["attachments"]) + +MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +PIPELINE_VERSION = 1 +_propagator = TraceContextTextMapPropagator() + + +def _require_member(chat_id: UUID, user_id: UUID): + chat = storage.get_chat(chat_id) + if chat is None or user_id not in chat.member_ids: + raise HTTPException(status_code=404, detail="not_found") + return chat + + +def _validate_filename(name: str) -> str: + if not name or len(name) > 255: + raise HTTPException(status_code=400, detail="invalid_request") + if name.strip() != name or name.strip() == "": + raise HTTPException(status_code=400, detail="invalid_request") + for ch in name: + if ch in ("/", "\\", "\x00") or unicodedata.category(ch).startswith("C"): + raise HTTPException(status_code=400, detail="invalid_request") + return name + + +def _detect_content_type(data: bytes) -> Optional[str]: + """MIME по содержимому, не по заголовку.""" + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + return None + + +def _validate_image(data: bytes) -> str: + """MIME по содержимому + лимит пикселей + полная проверка декодирования.""" + ctype = _detect_content_type(data) + if ctype is None: + raise HTTPException(status_code=415, detail="unsupported_media_type") + try: + img = Image.open(io.BytesIO(data)) + width, height = img.size + if width <= 0 or height <= 0: + raise HTTPException(status_code=415, detail="unsupported_media_type") + if width * height > MAX_IMAGE_PIXELS: + raise HTTPException(status_code=413, detail="payload_too_large") + img.load() + except HTTPException: + raise + except Image.DecompressionBombError: + raise HTTPException(status_code=413, detail="payload_too_large") + except Exception: + raise HTTPException(status_code=415, detail="unsupported_media_type") + return ctype + + +@router.post( + path="/chats/{chat_id}/attachments", + summary="Принять один PNG/JPEG, до 10 MiB (lab5: 202 queued)", + response_model=Attachment, + status_code=202, +) +async def upload_attachment( + chat_id: UUID, + request: Request, + user: User = Depends(current_user), + filename: str = Query(...), +) -> Attachment: + _require_member(chat_id, user.id) + safe_name = _validate_filename(filename) + + cl = request.headers.get("content-length") + if cl is not None: + try: + if int(cl) > MAX_UPLOAD_BYTES: + raise HTTPException(status_code=413, detail="payload_too_large") + except ValueError: + raise HTTPException(status_code=400, detail="invalid_request") + + chunks: list[bytes] = [] + total = 0 + async for chunk in request.stream(): + if not chunk: + continue + total += len(chunk) + if total > MAX_UPLOAD_BYTES: + raise HTTPException(status_code=413, detail="payload_too_large") + chunks.append(chunk) + body = b"".join(chunks) + if not body: + raise HTTPException(status_code=400, detail="invalid_request") + + content_type = await run_in_threadpool(_validate_image, body) + sha256 = hashlib.sha256(body).hexdigest() + + attachment_id = uuid4() + staging_key = s3.staging_key(chat_id, attachment_id) + + # 1) Durable staging в S3 (приватный prefix). + try: + await run_in_threadpool(s3.put_bytes, staging_key, body, content_type) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=503, detail="unavailable") from exc + + now = datetime.now(timezone.utc) + att = Attachment( + id=attachment_id, + chat_id=chat_id, + owner_id=user.id, + filename=safe_name, + content_type=content_type, + size_bytes=len(body), + sha256=sha256, + status="queued", + variants=[], + error_code=None, + created_at=now, + updated_at=now, + ) + + job_id = uuid4() + + # lab6: снимок W3C trace context активного HTTP span. + trace_carrier: dict[str, str] = {} + try: + _propagator.inject(trace_carrier) + except Exception: + trace_carrier = {} + + outbox_payload = { + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": PIPELINE_VERSION, + "created_at": now.isoformat().replace("+00:00", "Z"), + } + if trace_carrier.get("traceparent"): + outbox_payload["traceparent"] = trace_carrier["traceparent"] + if trace_carrier.get("tracestate"): + outbox_payload["tracestate"] = trace_carrier["tracestate"] + + # 2) INSERT attachment + outbox в ОДНОЙ транзакции. + # При сбое — компенсация: удаляем staging. + try: + await run_in_threadpool( + storage.create_attachment_queued, att, staging_key, outbox_payload + ) + except Exception: + try: + await run_in_threadpool(s3.delete_object, staging_key) + except Exception: + pass + raise HTTPException(status_code=503, detail="unavailable") + + return att + + +@router.get( + path="/attachments/{attachment_id}", + summary="Метаданные вложения, доступ участникам чата", + response_model=Attachment, +) +def get_attachment( + attachment_id: UUID, + user: User = Depends(current_user), +) -> Attachment: + pair = storage.get_attachment_with_key(attachment_id) + if pair is None: + raise HTTPException(status_code=404, detail="not_found") + att, _key = pair + _require_member(att.chat_id, user.id) + return att + + +@router.get( + path="/attachments/{attachment_id}/content", + summary="Скачать байты (original или variant) через авторизованный API", +) +def download_attachment( + attachment_id: UUID, + user: User = Depends(current_user), + variant: str = Query("original"), +): + if variant not in ("original", "thumbnail", "crop", "optimized"): + raise HTTPException(status_code=400, detail="invalid_request") + + pair = storage.get_attachment_with_key(attachment_id) + if pair is None: + raise HTTPException(status_code=404, detail="not_found") + att, object_key = pair + _require_member(att.chat_id, user.id) + if att.status != "ready": + raise HTTPException(status_code=409, detail="conflict") + + if variant == "original": + key = object_key + ctype_expected = att.content_type + else: + vs = [v for v in att.variants if v.name == variant] + if not vs: + raise HTTPException(status_code=404, detail="not_found") + v = vs[0] + key = s3.variant_key(att.chat_id, att.id, variant) + ctype_expected = v.content_type + + try: + data, ctype = s3.get_object(key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in ("404", "NoSuchKey", "NotFound"): + raise HTTPException(status_code=404, detail="not_found") + raise HTTPException(status_code=503, detail="unavailable") from exc + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=503, detail="unavailable") from exc + + return Response( + content=data, + media_type=ctype or ctype_expected, + ) \ No newline at end of file diff --git a/src/app/api/auth.py b/src/app/api/auth.py new file mode 100644 index 0000000..03c57a6 --- /dev/null +++ b/src/app/api/auth.py @@ -0,0 +1,81 @@ +import base64 +import binascii +import os + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.security import HTTPBasic, HTTPBasicCredentials + +from ..models.schemas import Session +from ..storage.postgres import storage + + +router = APIRouter(prefix="/api/v1/auth", tags=["auth"]) + +DEFAULT_TTL_SECONDS = 3600 + + +def _ttl_seconds() -> int: + try: + return int(os.environ.get("SESSION_TTL_SECONDS", DEFAULT_TTL_SECONDS)) + except ValueError: + return DEFAULT_TTL_SECONDS + + +def _parse_basic_header(request: Request) -> tuple[str, str] | None: + """Ручной разбор Basic: base64 -> UTF-8 -> split по первому ':'. + + HTTPBasic из Starlette декодирует latin-1 и ломает кириллицу. + Здесь пароль читается как UTF-8 (RFC 7617 разрешает UTF-8 charset). + Username остаётся ASCII по контракту. + """ + raw = request.headers.get("Authorization", "") + if not raw.lower().startswith("basic "): + return None + try: + decoded = base64.b64decode(raw[6:].strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + if ":" not in decoded: + return None + username, password = decoded.split(":", 1) + if not username or not password: + return None + try: + username.encode("ascii") + except UnicodeEncodeError: + return None + return username, password + + +@router.post( + path="/sessions", + summary="Вход: Basic username:password; выдаёт сессию", + response_model=Session, + status_code=201, +) +def create_session(request: Request) -> Session: + parsed = _parse_basic_header(request) + if parsed is None: + raise HTTPException(status_code=401, detail="unauthorized") + username, password = parsed + user = storage.verify_credentials(username, password) + if user is None: + raise HTTPException(status_code=401, detail="unauthorized") + token, expires_at = storage.create_session(user.id, _ttl_seconds()) + return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user) + + +@router.delete( + path="/sessions/current", + summary="Отозвать текущую сессию", + status_code=204, +) +def revoke_session(request: Request) -> None: + auth = request.headers.get("Authorization", "") + if not auth.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="unauthorized") + token = auth[7:].strip() + if not token or storage.get_session_user(token) is None: + raise HTTPException(status_code=401, detail="unauthorized") + storage.revoke_session(token) + return None \ No newline at end of file diff --git a/src/app/api/chats.py b/src/app/api/chats.py new file mode 100644 index 0000000..0384bdf --- /dev/null +++ b/src/app/api/chats.py @@ -0,0 +1,94 @@ +import base64 +import json +from datetime import datetime +from uuid import UUID +from fastapi import APIRouter, Depends, HTTPException, Query +from ..models.schemas import Chat, ChatPage, CreateChat, User +from ..storage.postgres import storage +from .security import current_user + + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +def _encode_cursor(user_id: UUID, chat: Chat) -> str: + """Простой курсор: user_id + id + created_at, base64.""" + data = {"user_id": str(user_id), "id": str(chat.id), "created_at": chat.created_at.isoformat()} + return base64.urlsafe_b64encode(json.dumps(data).encode()).decode() + + +def _decode_cursor(user_id: UUID, cursor: str) -> tuple[datetime, UUID]: + """Курсор привязан к пользователю. Любая порча → 400, не 500.""" + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + data = json.loads(raw) + except Exception: + raise HTTPException(status_code=400, detail="invalid_request") + if not isinstance(data, dict): + raise HTTPException(status_code=400, detail="invalid_request") + if data.get("user_id") != str(user_id): + raise HTTPException(status_code=400, detail="invalid_request") + try: + created_at = datetime.fromisoformat(data["created_at"]) + chat_id = UUID(data["id"]) + except (KeyError, ValueError, TypeError): + raise HTTPException(status_code=400, detail="invalid_request") + return created_at, chat_id + + +@router.post( + path="", + summary="Создать чат", + response_model=Chat, + status_code=201, +) +def create_chat( + payload: CreateChat, + user: User = Depends(current_user), +) -> Chat: + """Вызывающий должен входить в member_ids""" + user_id = user.id + if user_id not in payload.member_ids: + raise HTTPException(status_code=400, detail="invalid_request") + if len(set(payload.member_ids)) != len(payload.member_ids): + raise HTTPException(status_code=400, detail="invalid_request") + try: + return storage.create_chat(payload.title, list(payload.member_ids)) + except ValueError: + raise HTTPException(status_code=400, detail="invalid_request") + + +@router.get( + path="", + summary="Получить список чатов текущего пользователя", + response_model=ChatPage, +) +def list_chats( + user: User = Depends(current_user), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = None, +) -> ChatPage: + user_id = user.id + chats = storage.list_chats_for_user(user_id) + if cursor: + ccreated, cid = _decode_cursor(user_id, cursor) + ccreated_iso = ccreated.isoformat() + chats = [c for c in chats if (c.created_at.isoformat(), str(c.id)) > (ccreated_iso, str(cid))] + page = chats[:limit] + next_cursor = _encode_cursor(user_id, page[-1]) if len(chats) > limit else None + return ChatPage(items=page, next_cursor=next_cursor) + + +@router.get( + path="/{chat_id}", + summary="Получить доступный чат по идентификатору", + response_model=Chat, +) +def get_chat( + chat_id: UUID, + user: User = Depends(current_user), +) -> Chat: + chat = storage.get_chat(chat_id) + if chat is None or user.id not in chat.member_ids: + raise HTTPException(status_code=404, detail="not_found") + return chat \ No newline at end of file diff --git a/src/app/api/health.py b/src/app/api/health.py new file mode 100644 index 0000000..908b481 --- /dev/null +++ b/src/app/api/health.py @@ -0,0 +1,52 @@ +from fastapi import APIRouter +from ..models.schemas import Health +from ..storage import db as _db +from ..storage import s3 as _s3 +from ..storage.sessions import _redis_client, _store + + +router = APIRouter(tags=["health"]) + + +@router.get( + path="/health/live", + summary="Проверка жизнеспособности процесса", + response_model=Health, +) +def liveness() -> Health: + """Процесс жив""" + return Health(status="ok") + + +@router.get( + path="/health/ready", + summary="Проверка готовности принимать трафик", + response_model=Health, +) +def readiness() -> Health: + """200, если PostgreSQL, Redis (в режиме redis) и S3 доступны. + + При недоступности зависимости выбрасываем RuntimeError("*_unavailable"), + main.py конвертирует его в 503 с телом Error из contracts/openapi.json. + Все проверки — с короткими таймаутами, чтобы не зависать надолго. + """ + try: + with _db.connection() as conn, conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() + except Exception as exc: # noqa: BLE001 + # Пул уже проверяет живость соединения (check_connection) и короткий + # timeout=3.0; здесь лишь превращаем любую ошибку БД в 503, чтобы + # после рестарта PostgreSQL readiness быстро возвращался в 200. + raise RuntimeError("db_unavailable: %s" % type(exc).__name__) from exc + # Redis (в режиме redis) + if _store() == "redis": + _redis_client().ping() + # S3: HEAD несуществующего ключа. 404/None → S3 доступен. + # Короткий таймаут клиента гарантирует быстрый ответ, а ошибка + # преобразуется в RuntimeError("s3_unavailable") → 503. + try: + _s3.head_object("attachments/__health_probe__") + except Exception as exc: # noqa: BLE001 + raise RuntimeError("s3_unavailable: %s" % type(exc).__name__) from exc + return Health(status="ok") \ No newline at end of file diff --git a/src/app/api/messages.py b/src/app/api/messages.py new file mode 100644 index 0000000..efc787a --- /dev/null +++ b/src/app/api/messages.py @@ -0,0 +1,141 @@ +import base64 +import hashlib +import json +from datetime import datetime +from uuid import UUID +from fastapi import APIRouter, Depends, Header, HTTPException, Query +from ..models.schemas import CreateMessage, Message, MessagePage, User +from ..storage.postgres import storage +from .security import current_user + + +router = APIRouter(prefix="/api/v1/chats", tags=["messages"]) + +# Контракт: Idempotency-Key ASCII [A-Za-z0-9._:-]{1,128}. +_IDEMPOTENCY_ALLOWED = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._:-") + + +def _require_member(chat_id: UUID, user_id: UUID): + chat = storage.get_chat(chat_id) + if chat is None or user_id not in chat.member_ids: + raise HTTPException(status_code=404, detail="not_found") + return chat + + +def _encode_cursor(chat_id: UUID, message: Message) -> str: + data = {"chat_id": str(chat_id), "id": str(message.id), + "created_at": message.created_at.isoformat()} + return base64.urlsafe_b64encode(json.dumps(data).encode()).decode() + + +def _decode_cursor(chat_id: UUID, cursor: str) -> tuple: + try: + data = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode()) + if data.get("chat_id") != str(chat_id): + raise ValueError("cursor_chat_mismatch") + return datetime.fromisoformat(data["created_at"]), UUID(data["id"]) + except Exception: + raise HTTPException(status_code=400, detail="invalid_request") + + +def _canonical_body_hash(payload: CreateMessage) -> str: + """sha256 канонизированного JSON. + + Поле attachment_ids присутствует в модели со значением по умолчанию [], + поэтому payload {"text":"x"} и {"text":"x","attachment_ids":[]} дают + одинаковый model_dump и одинаковый хэш — как требует контракт. + """ + data = payload.model_dump(mode="json") + canonical = json.dumps(data, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +@router.post( + path="/{chat_id}/messages", + summary="Отправить сообщение в чат", + response_model=Message, + status_code=201, +) +def create_message( + chat_id: UUID, + payload: CreateMessage, + user: User = Depends(current_user), + idempotency_key: str = Header(..., alias="Idempotency-Key"), +) -> Message: + # Строго ASCII, как требует контракт: isalnum() пропускает кириллицу. + if not (1 <= len(idempotency_key) <= 128): + raise HTTPException(status_code=400, detail="invalid_request") + if not all(c in _IDEMPOTENCY_ALLOWED for c in idempotency_key): + raise HTTPException(status_code=400, detail="invalid_request") + + _require_member(chat_id, user.id) + + text = payload.text or "" + attachment_ids = list(payload.attachment_ids) + + # Контракт: text (непустой) ИЛИ attachment_ids (>=1). + if text.strip() == "" and not attachment_ids: + raise HTTPException(status_code=400, detail="invalid_request") + + # Все вложения должны принадлежать этому чату. + if attachment_ids: + found = storage.list_attachments_by_ids(attachment_ids, chat_id) + if len(found) != len(attachment_ids): + raise HTTPException(status_code=404, detail="not_found") + for aid in attachment_ids: + att, _key = found[aid] + if att.status == "failed": + raise HTTPException(status_code=409, detail="conflict") + + method = "POST" + path = "/api/v1/chats/%s/messages" % chat_id + body_hash = _canonical_body_hash(payload) + + # Атомарная операция: INSERT ключа + INSERT сообщения + UPDATE ключа + # в ОДНОЙ транзакции. Сбой между ними невозможен: либо всё, либо ничего. + state, message, info = storage.add_message_idempotent( + chat_id=chat_id, + sender_id=user.id, + text=text, + attachment_ids=attachment_ids, + user_id=user.id, + method=method, + path=path, + key=idempotency_key, + body_hash=body_hash, + ) + + if state == "created": + return message + + if state == "retry": + raise HTTPException(status_code=503, detail="unavailable") + + # state == "replay" + if info["body_hash"] != body_hash: + raise HTTPException(status_code=409, detail="conflict") + if info["response_status"] == 0: + # Строка от старой версии кода: не зависаем, отдаём 503. + raise HTTPException(status_code=503, detail="unavailable") + return Message.model_validate(info["response_body"]) + + +@router.get( + path="/{chat_id}/messages", + summary="Получить историю сообщений чата", + response_model=MessagePage, +) +def list_messages( + chat_id: UUID, + user: User = Depends(current_user), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = None, +) -> MessagePage: + """Keyset-пагинация в SQL: (created_at, id) по возрастанию""" + _require_member(chat_id, user.id) + after = _decode_cursor(chat_id, cursor) if cursor else None + rows = storage.list_messages_page(chat_id, limit + 1, after=after) + has_more = len(rows) > limit + items = rows[:limit] + next_cursor = _encode_cursor(chat_id, items[-1]) if has_more and items else None + return MessagePage(items=items, next_cursor=next_cursor) \ No newline at end of file diff --git a/src/app/api/security.py b/src/app/api/security.py new file mode 100644 index 0000000..fc09938 --- /dev/null +++ b/src/app/api/security.py @@ -0,0 +1,24 @@ +from uuid import UUID + +from fastapi import HTTPException, Security +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from ..models.schemas import User +from ..storage.postgres import storage + +bearer_scheme = HTTPBearer(auto_error=False, scheme_name="bearerAuth") + + +def current_user( + credentials: HTTPAuthorizationCredentials | None = Security(bearer_scheme), +) -> User: + """Достаёт Bearer-токен и возвращает владельца сессии. + + Basic на бизнес-endpoint не принимается, X-User-Id игнорируется. + """ + if credentials is None or credentials.scheme.lower() != "bearer": + raise HTTPException(status_code=401, detail="unauthorized") + user = storage.get_session_user(credentials.credentials) + if user is None: + raise HTTPException(status_code=401, detail="unauthorized") + return user \ No newline at end of file diff --git a/src/app/api/users.py b/src/app/api/users.py new file mode 100644 index 0000000..88fd62c --- /dev/null +++ b/src/app/api/users.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter, Depends, HTTPException +from ..models.schemas import CreateUser, User +from ..storage.postgres import storage +from .security import current_user + +router = APIRouter(prefix="/api/v1", tags=["users"]) + + +@router.post( + path="/users", + summary="Создать пользователя", + response_model=User, + status_code=201, +) +def register_user(payload: CreateUser) -> User: + """Публичная регистрация. Пробельные display_name/password отсекаются + валидаторами схемы; сюда попадают только валидные значения.""" + try: + return storage.create_user(payload.username, payload.display_name, payload.password) + except ValueError: + raise HTTPException(status_code=409, detail="conflict") + + +@router.get( + path="/users/me", + summary="Получить текущего пользователя", + response_model=User, +) +def current_user_me(user: User = Depends(current_user)) -> User: + """Текущий пользователь по Bearer-сессии.""" + return user \ No newline at end of file diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 0000000..6514053 --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,269 @@ +import logging +import os +import signal +import sys +import time +import uuid +from contextlib import asynccontextmanager +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.openapi.utils import get_openapi +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException +import psycopg +import uvicorn +from opentelemetry import trace as _otel_trace +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from .api import attachments, auth, chats, health, messages, users +from .storage.db import init_pool, close_pool +from .telemetry import init_telemetry, INSTANCE_ID, meter, tracer + + +# Базовый логгер: реальный формат выставляет telemetry.install_log_format() +# при вызове init_telemetry(). Здесь — минимальный fallback для импорта. +logging.basicConfig(level=logging.INFO, stream=sys.stdout) +logger = logging.getLogger("app") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + init_pool() + logger.info( + "event=startup app_env=%s session_store=%s instance_id=%s", + os.environ.get("APP_ENV", "lab"), + os.environ.get("SESSION_STORE", "postgres"), + INSTANCE_ID, + ) + yield + close_pool() + logger.info("event=shutdown_complete instance_id=%s", INSTANCE_ID) + + +app = FastAPI( + title="MTUCI Messenger — lab6", + version="1.6.0", + lifespan=lifespan, + swagger_ui_parameters={"persistAuthorization": True}, +) + +# OTel инициализация и FastAPI auto-instrumentation. +init_telemetry("messenger-api", "1.6.0") + + +# --- Метрики HTTP --- +_http_counter = meter("messenger").create_counter( + "http_requests_total", + description="HTTP requests count", + unit="1", +) +# Явные границы бакетов в СЕКУНДАХ, с разрешением в миллисекундном диапазоне +# и вокруг порога SLO 2s (upload p95 alert). Дефолтные экспоненциальные +# бакеты (0, 5, 10, 25, …) грубы: миллисекундные запросы попадали в один +# широкий bucket, а histogram_quantile интерполировал ложные секунды. +_HTTP_DURATION_BUCKETS = [ + 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05, + 0.075, 0.1, 0.15, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, +] +_http_hist = meter("messenger").create_histogram( + "http_request_duration_seconds", + description="HTTP request duration", + unit="s", + explicit_bucket_boundaries_advisory=_HTTP_DURATION_BUCKETS, +) + +# Фильтр: убираем реальные UUID из маршрута → route template. +import re as _re +_ROUTE_UUID_RE = _re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") + + +def _route_template(path: str) -> str: + return _ROUTE_UUID_RE.sub("{id}", path) + + +def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + schema = get_openapi(title=app.title, version=app.version, routes=app.routes) + app.openapi_schema = schema + return schema + + +app.openapi = custom_openapi + +# Отдельный tracer для событийного span в middleware. Он создаётся как child +# SERVER span, который создаёт OTel FastAPI instrumentation ниже. +# ВАЖНО: FastAPIInstrumentor.instrument_app(app) вызывается ПОСЛЕ объявления +# middleware, поэтому OTel ASGI middleware оказывается внешним: он извлекает +# W3C traceparent из входящих headers, продолжает внешний trace и создаёт +# SERVER span. Наш middleware выполняется внутри него, создаёт http.request +# как INTERNAL child, и логи/метрики берут корректный trace_id. +_http_tracer = tracer("app.http") + + +@app.middleware("http") +async def observability_middleware(request: Request, call_next): + """X-Request-Id и X-Instance-Id во всех ответах + структурированный лог + метрики. + + Лог event=http пишется ВНУТРИ активного span (`http.request`), он + child SERVER span от OTel auto-instrumentation, поэтому trace_id + корректно связывает лог с внешним trace. + """ + request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = request_id + started = time.monotonic() + route = _route_template(request.url.path) + + # http.request — INTERNAL child текущего (SERVER) span, созданного OTel + # auto-instrumentation. Parent берётся из активного контекста: входящий + # traceparent уже извлечён ASGI middleware. + with _http_tracer.start_as_current_span( + "http.request", + attributes={ + "http.method": request.method, + "http.route": route, + }, + ) as span: + response = await call_next(request) + duration_s = time.monotonic() - started + duration_ms = round(duration_s * 1000, 2) + status_class = "%dxx" % (response.status_code // 100) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + + _http_counter.add( + 1, + {"http.method": request.method, "http.route": route, + "http.status_class": status_class}, + ) + _http_hist.record( + duration_s, + {"http.method": request.method, "http.route": route, + "http.status_class": status_class}, + ) + span.set_attribute("http.status_class", status_class) + span.set_attribute("http.status_code", response.status_code) + + # Лог ВНУТРИ span → record.trace_id валидный, Loki получает + # корреляцию с Tempo. + logger.info( + "event=http method=%s path=%s status=%s duration_ms=%s " + "request_id=%s instance_id=%s", + request.method, request.url.path, response.status_code, + duration_ms, request_id, INSTANCE_ID, + ) + return response + + +# FastAPI auto-instrumentation ДОЛЖНА быть установлена ПОСЛЕ пользовательского +# @app.middleware("http"), чтобы оказаться внешним middleware и создать SERVER +# span с корректным parent из входного traceparent. Этот вызов также +# инструментирует httpx/requests client-вызовы внутри приложения. +FastAPIInstrumentor.instrument_app(app) + + +# --- Обработчики ошибок (без изменений) --- +_ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + + +def _unauthorized_challenge(path: str) -> str: + if path == "/api/v1/auth/sessions": + return 'Basic realm="messenger", charset="UTF-8"' + return 'Bearer realm="messenger"' + + +def _error_response(status_code: int, message: str, request_id: str, + path: str = "") -> JSONResponse: + code = _ERROR_CODES.get(status_code, "internal_error") + headers = {} + if status_code == 401: + headers["WWW-Authenticate"] = _unauthorized_challenge(path) + headers["X-Request-Id"] = request_id + headers["X-Instance-Id"] = INSTANCE_ID + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + request_id = getattr(request.state, "request_id", uuid.uuid4().hex) + message = exc.detail if isinstance(exc.detail, str) else "Ошибка" + return _error_response(exc.status_code, message, request_id, request.url.path) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + request_id = getattr(request.state, "request_id", uuid.uuid4().hex) + return _error_response(400, "Неверный запрос", request_id, request.url.path) + + +@app.exception_handler(psycopg.OperationalError) +async def db_unavailable_handler(request: Request, exc: psycopg.OperationalError): + request_id = getattr(request.state, "request_id", uuid.uuid4().hex) + logger.error("event=db_unavailable type=%s request_id=%s", + type(exc).__name__, request_id) + return _error_response(503, "Зависимость недоступна", request_id, request.url.path) + + +@app.exception_handler(RuntimeError) +async def runtime_error_handler(request: Request, exc: RuntimeError): + request_id = getattr(request.state, "request_id", uuid.uuid4().hex) + text = str(exc) + if "db_unavailable" in text: + logger.error("event=db_unavailable request_id=%s", request_id) + return _error_response(503, "Зависимость недоступна", request_id, request.url.path) + if "redis_unavailable" in text: + logger.error("event=redis_unavailable request_id=%s", request_id) + return _error_response(503, "Зависимость недоступна", request_id, request.url.path) + if "s3_unavailable" in text: + logger.error("event=s3_unavailable request_id=%s", request_id) + return _error_response(503, "Зависимость недоступна", request_id, request.url.path) + logger.error("event=runtime_error type=%s request_id=%s", + type(exc).__name__, request_id) + return _error_response(500, "Внутренняя ошибка", request_id, request.url.path) + + +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): + request_id = getattr(request.state, "request_id", uuid.uuid4().hex) + logger.error("event=unhandled_exception type=%s request_id=%s", + type(exc).__name__, request_id) + return _error_response(500, "Внутренняя ошибка", request_id, request.url.path) + + +app.include_router(health.router) +app.include_router(auth.router) +app.include_router(users.router) +app.include_router(chats.router) +app.include_router(messages.router) +app.include_router(attachments.router) + + +if __name__ == "__main__": + config = uvicorn.Config( + "app.main:app", + host="0.0.0.0", + port=8080, + log_level="info", + ) + server = uvicorn.Server(config) + + def _handle_sigterm(signum, frame): + logger.info("event=sigterm_received signal=%s", signum) + server.should_exit = True + + signal.signal(signal.SIGTERM, _handle_sigterm) + server.run() \ No newline at end of file diff --git a/src/app/models/__init__.py b/src/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/models/schemas.py b/src/app/models/schemas.py new file mode 100644 index 0000000..5e1a290 --- /dev/null +++ b/src/app/models/schemas.py @@ -0,0 +1,163 @@ +from datetime import datetime, timezone +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator + + +def _to_z(dt: datetime) -> str: + """RFC 3339 UTC с Z, как требует contracts/README.md.""" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + dt = dt.astimezone(timezone.utc) + return dt.isoformat().replace("+00:00", "Z") + + +def _not_blank(value: str) -> str: + """Контракт: обязательные строки не состоят только из пробельных символов.""" + if value is None or value.strip() == "": + raise ValueError("must not be blank") + return value + + +class Health(BaseModel): + model_config = ConfigDict(extra="forbid") + status: str + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def _display_name_not_blank(cls, v: str) -> str: + return _not_blank(v) + + @field_validator("password") + @classmethod + def _password_not_blank(cls, v: str) -> str: + return _not_blank(v) + + +class User(BaseModel): + model_config = ConfigDict(extra="forbid") + id: UUID + username: str + display_name: str + created_at: datetime + + @field_serializer("created_at") + def _ser_created_at(self, dt: datetime) -> str: + return _to_z(dt) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: List[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def _title_not_blank(cls, v: str) -> str: + return _not_blank(v) + + +class Chat(BaseModel): + model_config = ConfigDict(extra="forbid") + id: UUID + title: str + member_ids: List[UUID] + created_at: datetime + + @field_serializer("created_at") + def _ser_created_at(self, dt: datetime) -> str: + return _to_z(dt) + + +class CreateMessage(BaseModel): + """text может быть пустым, если есть хотя бы одно вложение (lab4).""" + model_config = ConfigDict(extra="forbid") + text: str = Field(default="", max_length=4000) + attachment_ids: List[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def _unique_attachments(cls, v: List[UUID]) -> List[UUID]: + if len(set(v)) != len(v): + raise ValueError("duplicate attachment_ids") + return v + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: List[UUID] = Field(default_factory=list) + + @field_serializer("created_at") + def _ser_created_at(self, dt: datetime) -> str: + return _to_z(dt) + + +class ChatPage(BaseModel): + model_config = ConfigDict(extra="forbid") + items: List[Chat] + next_cursor: Optional[str] = None + + +class MessagePage(BaseModel): + model_config = ConfigDict(extra="forbid") + items: List[Message] + next_cursor: Optional[str] = None + + +class Session(BaseModel): + model_config = ConfigDict(extra="forbid") + token: str + token_type: str = "Bearer" + expires_at: datetime + user: User + + @field_serializer("expires_at") + def _ser_expires_at(self, dt: datetime) -> str: + return _to_z(dt) + + +class Variant(BaseModel): + model_config = ConfigDict(extra="forbid") + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + model_config = ConfigDict(extra="forbid") + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str # "ready" + variants: List[Variant] = Field(default_factory=list) + error_code: Optional[str] = None + created_at: datetime + updated_at: datetime + + @field_serializer("created_at") + def _ser_created_at(self, dt: datetime) -> str: + return _to_z(dt) + + @field_serializer("updated_at") + def _ser_updated_at(self, dt: datetime) -> str: + return _to_z(dt) \ No newline at end of file diff --git a/src/app/outbox_worker.py b/src/app/outbox_worker.py new file mode 100644 index 0000000..5021783 --- /dev/null +++ b/src/app/outbox_worker.py @@ -0,0 +1,128 @@ +"""Outbox worker lab5/lab6: publish неопубликованных job в RabbitMQ с трейсингом. + +lab6: payload строки outbox содержит traceparent/tracestate, снятые в момент +upload-запроса API. Восстанавливаем контекст, создаём span job.publish как +child upload-span и инжектим обновлённый traceparent в headers сообщения. + +Лог event=outbox_published пишется ВНУТРИ span job.publish, чтобы в Loki +попадал настоящий trace_id (корреляция лог↔trace). +""" +import logging +import signal +import sys +import time +import traceback + +from opentelemetry import context as otel_context, trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from .storage.db import init_pool, close_pool +from .storage.postgres import storage +from .storage.rabbit import QUEUE, publish_job +from .telemetry import init_telemetry + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) +log = logging.getLogger("outbox") + +_stop = False +_tracer = trace.get_tracer("outbox") +_propagator = TraceContextTextMapPropagator() + +init_telemetry("messenger-outbox", "1.6.0") + + +def _handle_sigterm(signum, frame): + global _stop + log.info("event=sigterm_received signal=%s", signum) + _stop = True + + +def _extract_parent(payload: dict) -> otel_context.Context: + carrier: dict[str, str] = {} + if payload.get("traceparent"): + carrier["traceparent"] = str(payload["traceparent"]) + if payload.get("tracestate"): + carrier["tracestate"] = str(payload["tracestate"]) + if not carrier: + return otel_context.Context() + try: + return _propagator.extract(carrier=carrier) + except Exception as exc: # noqa: BLE001 + log.warning("event=trace_parent_parse_failed type=%s", type(exc).__name__) + return otel_context.Context() + + +def _publish_with_trace(payload: dict, outbox_id: int) -> None: + """Публикует payload; mark_published и лог — внутри span.""" + parent_ctx = _extract_parent(payload) + carrier: dict[str, str] = {} + with _tracer.start_as_current_span( + "job.publish", + context=parent_ctx, + kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "job.id": str(payload.get("job_id", "")), + "job.type": str(payload.get("type", "unknown")), + "outbox.id": int(outbox_id), + }, + ) as span: + try: + try: + _propagator.inject(carrier) + except Exception as exc: # noqa: BLE001 + log.warning("event=trace_inject_failed type=%s", type(exc).__name__) + carrier = {} + publish_job(payload, extra_headers=carrier) + storage.mark_outbox_published(outbox_id) + # Лог внутри span → record.trace_id валидный. + log.info("event=outbox_published id=%s job_id=%s", + outbox_id, payload.get("job_id")) + span.set_status(Status(StatusCode.OK)) + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + raise + + +def main() -> None: + signal.signal(signal.SIGTERM, _handle_sigterm) + init_pool() + log.info("event=outbox_start") + + while not _stop: + try: + rows = storage.list_unpublished_outbox(limit=50) + except Exception as exc: # noqa: BLE001 + log.error("event=outbox_select_failed type=%s msg=%s", + type(exc).__name__, str(exc)[:300]) + time.sleep(2) + continue + + for outbox_id, payload in rows: + if _stop: + break + try: + _publish_with_trace(payload, outbox_id) + except Exception as exc: # noqa: BLE001 + log.warning( + "event=outbox_publish_failed id=%s type=%s msg=%s", + outbox_id, type(exc).__name__, str(exc)[:300], + ) + time.sleep(2) + break + + time.sleep(2) + + close_pool() + log.info("event=outbox_stop") + + +if __name__ == "__main__": + try: + main() + except Exception: + log.error("event=outbox_fatal\n%s", traceback.format_exc()) + sys.exit(1) \ No newline at end of file diff --git a/src/app/storage/__init__.py b/src/app/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/storage/db.py b/src/app/storage/db.py new file mode 100644 index 0000000..1348940 --- /dev/null +++ b/src/app/storage/db.py @@ -0,0 +1,212 @@ +import os +from contextlib import contextmanager +from typing import Iterator + +import psycopg +from psycopg_pool import ConnectionPool + + +_pool: ConnectionPool | None = None + + +def _dsn() -> str: + dsn = os.environ.get("DATABASE_URL") + if not dsn: + raise RuntimeError("DATABASE_URL is not set") + return dsn + + +def init_pool() -> None: + """Ограниченный пул соединений с таймаутами. Вызывается один раз на старте.""" + global _pool + if _pool is not None: + return + _pool = ConnectionPool( + conninfo=_dsn(), + min_size=2, + max_size=10, + # Короткий таймаут получения соединения из пула: readiness/бизнес-запрос + # при мёртвой БД должен быстро вернуть 503, а не висеть. + timeout=3.0, + max_idle=300.0, + # check: перед выдачей соединения проверяем его живость; мёртвые + # соединения после рестарта PostgreSQL отбрасываются, пул сам + # открывает новые — readiness восстанавливается без рестарта API. + check=ConnectionPool.check_connection, + # Раз в 30с фоновый worker пула добивает min_size живыми соединениями, + # чтобы после старта БД готовность вернулась в пределах окна теста. + reconnect_timeout=30.0, + kwargs={"options": "-c statement_timeout=5000"}, + open=True, + ) + + +def close_pool() -> None: + global _pool + if _pool is not None: + _pool.close() + _pool = None + + +@contextmanager +def connection() -> Iterator[psycopg.Connection]: + """Соединение из пула; autocommit=False для явных транзакций.""" + if _pool is None: + init_pool() + assert _pool is not None + with _pool.connection() as conn: + yield conn + + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_messages_chat_created + ON messages (chat_id, created_at, id); + +-- sessions остаются как fallback, если SESSION_STORE=postgres. +CREATE TABLE IF NOT EXISTS sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL +); + +-- lab3: общая идемпотентность POST /messages. +-- scope: user_id + method + path + key (см. contracts/README.md). +-- body_hash — sha256 канонизированного JSON без attachment_ids=[], чтобы +-- отличать повтор «того же» body (201, тот же Message) от «другого» (409). +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + body_hash TEXT NOT NULL, + response_status INT NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_keys (created_at); + +-- lab4: вложения +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL, -- queued/processing/ready/failed + error_code TEXT, + processing_until TIMESTAMPTZ, -- lease для worker (lab5) + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_attachments_chat + ON attachments (chat_id); + +-- связь сообщение↔вложение (порядок сохраняем) +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + position INT NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); +CREATE INDEX IF NOT EXISTS idx_msg_att_message + ON message_attachments (message_id, position); + +-- lab5: варианты обработки (идемпотентность по (attachment_id, pipeline_version, variant)) +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INT NOT NULL, + variant TEXT NOT NULL, + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INT NOT NULL, + height INT NOT NULL, + sha256 TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); + +-- lab5 level 4: transactional outbox (разрыв COMMIT → publish) +CREATE TABLE IF NOT EXISTS outbox ( + id BIGSERIAL PRIMARY KEY, + job_id UUID NOT NULL UNIQUE, + payload JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ +); +CREATE INDEX IF NOT EXISTS idx_outbox_unpublished + ON outbox (id) WHERE published_at IS NULL; +""" + + +def init_schema() -> None: + """Идемпотентная миграция: безопасно вызывать многократно. + + Вызывается ОДИН РАЗ отдельным one-shot сервисом migrate + (см. compose.yaml). Реплики API миграции больше не выполняют: + они ждут service_completed_successfully. + """ + with connection() as conn: + with conn.cursor() as cur: + cur.execute(SCHEMA_SQL) + conn.commit() + + +def run_migrations() -> None: + """Точка входа для one-shot migrate-контейнера. + + Поднимает пул, применяет схему, закрывает пул. Завершается с кодом 0 + при успехе — этого достаточно для service_completed_successfully. + """ + init_pool() + try: + init_schema() + finally: + close_pool() + + +if __name__ == "__main__": + # Запуск: python -m app.storage.db + # (в compose.yaml — command: ["python", "-m", "app.storage.db"]) + import logging + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + log = logging.getLogger("migrate") + log.info("event=migrations_start") + run_migrations() + log.info("event=migrations_done") \ No newline at end of file diff --git a/src/app/storage/memory.py b/src/app/storage/memory.py new file mode 100644 index 0000000..76a468b --- /dev/null +++ b/src/app/storage/memory.py @@ -0,0 +1,177 @@ +import hashlib +import hmac +import secrets +import threading +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional +from uuid import UUID, uuid4 + +from ..models.schemas import Chat, Message, User + + +def now_utc() -> datetime: + """Время сервера в UTC с суффиксом Z на выходе.""" + return datetime.now(timezone.utc) + + +# --- Password KDF (stdlib scrypt) --- +_SCRYPT_N = 2 ** 14 +_SCRYPT_R = 8 +_SCRYPT_P = 1 +_SCRYPT_DKLEN = 32 + + +def _hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + dk = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=_SCRYPT_N, + r=_SCRYPT_R, + p=_SCRYPT_P, + dklen=_SCRYPT_DKLEN, + ) + return "scrypt$%d$%d$%d$%s$%s" % ( + _SCRYPT_N, _SCRYPT_R, _SCRYPT_P, + salt.hex(), dk.hex(), + ) + + +def _verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_hex, dk_hex = stored.split("$") + if scheme != "scrypt": + return False + dk = hashlib.scrypt( + password.encode("utf-8"), + salt=bytes.fromhex(salt_hex), + n=int(n), r=int(r), p=int(p), + dklen=len(bytes.fromhex(dk_hex)), + ) + return hmac.compare_digest(dk.hex(), dk_hex) + except Exception: + return False + + +class MemoryStorage: + """Потокобезопасное In-memory хранилище пользователей, чатов, сообщений и сессий. Все данные исчезают при перезапуске процесса""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._users: Dict[UUID, User] = {} + self._usernames: Dict[str, UUID] = {} + self._passwords: Dict[UUID, str] = {} + self._chats: Dict[UUID, Chat] = {} + self._messages: Dict[UUID, List[Message]] = {} + # token -> (user_id, expires_at) + self._sessions: Dict[str, tuple] = {} + + # ---------- Пользователи ---------- + def create_user(self, username: str, display_name: str, password: str) -> User: + with self._lock: + if username in self._usernames: + raise ValueError("username_taken") + user = User( + id=uuid4(), + username=username, + display_name=display_name, + created_at=now_utc(), + ) + self._users[user.id] = user + self._usernames[username] = user.id + self._passwords[user.id] = _hash_password(password) + return user + + def get_user(self, user_id: UUID) -> Optional[User]: + with self._lock: + return self._users.get(user_id) + + def get_user_by_username(self, username: str) -> Optional[User]: + with self._lock: + uid = self._usernames.get(username) + return self._users.get(uid) if uid else None + + def verify_credentials(self, username: str, password: str) -> Optional[User]: + with self._lock: + uid = self._usernames.get(username) + if uid is None: + return None + stored = self._passwords.get(uid) + if not stored or not _verify_password(password, stored): + return None + return self._users.get(uid) + + # ---------- Сессии ---------- + def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple: + with self._lock: + token = secrets.token_urlsafe(32) + expires_at = now_utc() + timedelta(seconds=ttl_seconds) + self._sessions[token] = (user_id, expires_at) + return token, expires_at + + def get_session_user(self, token: str) -> Optional[User]: + with self._lock: + entry = self._sessions.get(token) + if entry is None: + return None + user_id, expires_at = entry + if expires_at <= now_utc(): + self._sessions.pop(token, None) + return None + return self._users.get(user_id) + + def revoke_session(self, token: str) -> None: + with self._lock: + self._sessions.pop(token, None) + + # ---------- Чаты ---------- + def create_chat(self, title: str, member_ids: List[UUID]) -> Chat: + with self._lock: + for uid in member_ids: + if uid not in self._users: + raise ValueError("unknown_member") + chat = Chat( + id=uuid4(), + title=title, + member_ids=list(member_ids), + created_at=now_utc(), + ) + self._chats[chat.id] = chat + self._messages[chat.id] = [] + return chat + + def get_chat(self, chat_id: UUID) -> Optional[Chat]: + with self._lock: + return self._chats.get(chat_id) + + def list_chats_for_user(self, user_id: UUID) -> List[Chat]: + with self._lock: + chats = [c for c in self._chats.values() if user_id in c.member_ids] + chats.sort(key=lambda c: (c.created_at, str(c.id))) + return chats + + # ---------- Сообщения ---------- + def add_message(self, chat_id: UUID, sender_id: UUID, text: str) -> Message: + """Улучшение Трека 1: список сообщений хранится уже отсортированным. + created_at монотонен в пределах одного процесса, поэтому append + сохраняет порядок (created_at, id) по возрастанию — сортировка на + чтении не нужна. При конкурентных вставках под RLock порядок + сохраняется, но монотонность часов не гарантируется контрактом.""" + with self._lock: + message = Message( + id=uuid4(), + chat_id=chat_id, + sender_id=sender_id, + text=text, + created_at=now_utc(), + ) + self._messages.setdefault(chat_id, []).append(message) + return message + + def list_messages(self, chat_id: UUID) -> List[Message]: + with self._lock: + # Без пересортировки: список уже в порядке (created_at, id). + return list(self._messages.get(chat_id, [])) + + +storage = MemoryStorage() # единственный экземпляр хранилища на процесс \ No newline at end of file diff --git a/src/app/storage/postgres.py b/src/app/storage/postgres.py new file mode 100644 index 0000000..a36b768 --- /dev/null +++ b/src/app/storage/postgres.py @@ -0,0 +1,614 @@ +import hashlib +import hmac +import json as _json +import secrets +import threading +from datetime import datetime, timedelta, timezone +from typing import List, Optional +from uuid import UUID, uuid4 + +from ..models.schemas import Attachment, Chat, Message, User, Variant +from .db import connection +from .sessions import ( + create_session as _sess_create, + get_session_user as _sess_get, + revoke_session as _sess_revoke, +) + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +# --- Password KDF (stdlib scrypt) --- +_SCRYPT_N = 2 ** 14 +_SCRYPT_R = 8 +_SCRYPT_P = 1 +_SCRYPT_DKLEN = 32 + + +def _hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + dk = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=_SCRYPT_N, r=_SCRYPT_R, p=_SCRYPT_P, dklen=_SCRYPT_DKLEN, + ) + return "scrypt$%d$%d$%d$%s$%s" % ( + _SCRYPT_N, _SCRYPT_R, _SCRYPT_P, salt.hex(), dk.hex(), + ) + + +def _verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_hex, dk_hex = stored.split("$") + if scheme != "scrypt": + return False + dk = hashlib.scrypt( + password.encode("utf-8"), + salt=bytes.fromhex(salt_hex), + n=int(n), r=int(r), p=int(p), + dklen=len(bytes.fromhex(dk_hex)), + ) + return hmac.compare_digest(dk.hex(), dk_hex) + except Exception: + return False + + +def _row_to_user(row) -> User: + return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3]) + + +def _row_to_chat(row, member_ids: List[UUID]) -> Chat: + return Chat(id=row[0], title=row[1], member_ids=member_ids, created_at=row[2]) + + +def _row_to_message(row, attachment_ids: List[UUID]) -> Message: + return Message( + id=row[0], chat_id=row[1], sender_id=row[2], + text=row[3], created_at=row[4], + attachment_ids=attachment_ids, + ) + + +def _row_to_attachment(row, variants: Optional[list] = None) -> Attachment: + # row: id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + # status, error_code, created_at, updated_at + return Attachment( + id=row[0], chat_id=row[1], owner_id=row[2], + filename=row[3], content_type=row[4], size_bytes=row[5], + sha256=row[6], status=row[7], variants=variants or [], + error_code=row[8], + created_at=row[9], updated_at=row[10], + ) + + +class PostgresStorage: + """Потокобезопасная обёртка; параллелизм обеспечивает PostgreSQL.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + + # ---------- Пользователи ---------- + def create_user(self, username: str, display_name: str, password: str) -> User: + with self._lock, connection() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM users WHERE username = %s", (username,)) + if cur.fetchone() is not None: + raise ValueError("username_taken") + user = User( + id=uuid4(), username=username, + display_name=display_name, created_at=now_utc(), + ) + try: + cur.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (%s, %s, %s, %s, %s)", + (user.id, user.username, user.display_name, + _hash_password(password), user.created_at), + ) + except Exception as exc: + if exc.__class__.__name__ == "UniqueViolation": + raise ValueError("username_taken") + raise + conn.commit() + return user + + def get_user(self, user_id: UUID) -> Optional[User]: + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ) + row = cur.fetchone() + return _row_to_user(row) if row else None + + def get_user_by_username(self, username: str) -> Optional[User]: + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, username, display_name, created_at FROM users WHERE username = %s", + (username,), + ) + row = cur.fetchone() + return _row_to_user(row) if row else None + + def verify_credentials(self, username: str, password: str) -> Optional[User]: + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, username, display_name, created_at, password_hash" + " FROM users WHERE username = %s", + (username,), + ) + row = cur.fetchone() + if row is None: + return None + if not _verify_password(password, row[4]): + return None + return _row_to_user(row[:4]) + + # ---------- Сессии ---------- + def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple: + return _sess_create(user_id, ttl_seconds) + + def get_session_user(self, token: str) -> Optional[User]: + return _sess_get(token) + + def revoke_session(self, token: str) -> None: + _sess_revoke(token) + + # ---------- Чаты ---------- + def create_chat(self, title: str, member_ids: List[UUID]) -> Chat: + with self._lock, connection() as conn: + with conn.cursor() as cur: + # проверяем всех участников + cur.execute( + "SELECT id FROM users WHERE id = ANY(%s)", + (list(member_ids),), + ) + found = {r[0] for r in cur.fetchall()} + if len(found) != len(set(member_ids)): + raise ValueError("unknown_member") + chat = Chat( + id=uuid4(), title=title, + member_ids=list(member_ids), created_at=now_utc(), + ) + cur.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat.id, chat.title, chat.created_at), + ) + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat.id, uid) for uid in member_ids], + ) + conn.commit() + return chat + + def get_chat(self, chat_id: UUID) -> Optional[Chat]: + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ) + row = cur.fetchone() + if row is None: + return None + cur.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id,), + ) + members = [r[0] for r in cur.fetchall()] + return _row_to_chat(row, members) + + def list_chats_for_user(self, user_id: UUID) -> List[Chat]: + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT c.id, c.title, c.created_at FROM chats c" + " JOIN chat_members m ON m.chat_id = c.id" + " WHERE m.user_id = %s ORDER BY c.created_at, c.id", + (user_id,), + ) + rows = cur.fetchall() + chats = [] + for row in rows: + cur.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s", (row[0],), + ) + members = [r[0] for r in cur.fetchall()] + chats.append(_row_to_chat(row, members)) + return chats + + # ---------- Сообщения ---------- + def _load_message_attachments(self, cur, message_ids: List[UUID]) -> dict[UUID, List[UUID]]: + if not message_ids: + return {} + cur.execute( + "SELECT message_id, attachment_id, position FROM message_attachments" + " WHERE message_id = ANY(%s) ORDER BY message_id, position", + (list(message_ids),), + ) + result: dict[UUID, List[UUID]] = {} + for mid, aid, _pos in cur.fetchall(): + result.setdefault(mid, []).append(aid) + return result + + def list_messages_page( + self, chat_id: UUID, limit: int, after: Optional[tuple] = None, + ) -> List[Message]: + with connection() as conn, conn.cursor() as cur: + if after is None: + cur.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages" + " WHERE chat_id = %s ORDER BY created_at, id LIMIT %s", + (chat_id, limit), + ) + else: + after_ts, after_id = after + cur.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages" + " WHERE chat_id = %s AND (created_at, id) > (%s, %s)" + " ORDER BY created_at, id LIMIT %s", + (chat_id, after_ts, after_id, limit), + ) + rows = cur.fetchall() + ids = [r[0] for r in rows] + att_map = self._load_message_attachments(cur, ids) + return [_row_to_message(r, att_map.get(r[0], [])) for r in rows] + + # ---------- Идемпотентность (lab3) ---------- + def add_message_idempotent( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + attachment_ids: List[UUID], + user_id: UUID, + method: str, + path: str, + key: str, + body_hash: str, + ) -> tuple[str, Optional[Message], Optional[dict]]: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO idempotency_keys" + " (user_id, method, path, key, body_hash, response_status, response_body)" + " VALUES (%s, %s, %s, %s, %s, 0, '{}'::jsonb)" + " ON CONFLICT (user_id, method, path, key) DO NOTHING" + " RETURNING key", + (user_id, method, path, key, body_hash), + ) + won = cur.fetchone() is not None + + if won: + message = Message( + id=uuid4(), chat_id=chat_id, sender_id=sender_id, + text=text, created_at=now_utc(), + attachment_ids=list(attachment_ids), + ) + cur.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " VALUES (%s, %s, %s, %s, %s)", + (message.id, message.chat_id, message.sender_id, + message.text, message.created_at), + ) + if attachment_ids: + cur.executemany( + "INSERT INTO message_attachments" + " (message_id, attachment_id, position)" + " VALUES (%s, %s, %s)", + [(message.id, aid, idx) + for idx, aid in enumerate(attachment_ids)], + ) + body_json = _json.loads(message.model_dump_json()) + cur.execute( + "UPDATE idempotency_keys" + " SET response_status = %s, response_body = %s" + " WHERE user_id = %s AND method = %s AND path = %s AND key = %s", + (201, _json.dumps(body_json), + user_id, method, path, key), + ) + conn.commit() + return ("created", message, None) + + cur.execute( + "SELECT body_hash, response_status, response_body" + " FROM idempotency_keys" + " WHERE user_id = %s AND method = %s AND path = %s AND key = %s", + (user_id, method, path, key), + ) + row = cur.fetchone() + conn.commit() + + if row is None: + return ("retry", None, None) + return ( + "replay", + None, + { + "body_hash": row[0], + "response_status": row[1], + "response_body": row[2], + }, + ) + + # ---------- Вложения ---------- + def create_attachment(self, attachment: Attachment, object_key: str) -> Attachment: + """lab4 fallback (синхронный ready). Не используется в lab5.""" + with connection() as conn, conn.cursor() as cur: + cur.execute( + "INSERT INTO attachments" + " (id, chat_id, owner_id, filename, content_type, size_bytes," + " sha256, object_key, status, error_code, created_at, updated_at)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + ( + attachment.id, attachment.chat_id, attachment.owner_id, + attachment.filename, attachment.content_type, + attachment.size_bytes, attachment.sha256, object_key, + attachment.status, attachment.error_code, + attachment.created_at, attachment.updated_at, + ), + ) + conn.commit() + return attachment + + def create_attachment_queued( + self, + attachment: Attachment, + staging_key: str, + outbox_payload: dict, + ) -> Attachment: + """lab5: attachment в queued + строка outbox в ОДНОЙ транзакции.""" + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO attachments" + " (id, chat_id, owner_id, filename, content_type, size_bytes," + " sha256, object_key, status, error_code, processing_until," + " created_at, updated_at)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + ( + attachment.id, attachment.chat_id, attachment.owner_id, + attachment.filename, attachment.content_type, + attachment.size_bytes, attachment.sha256, staging_key, + "queued", None, None, + attachment.created_at, attachment.updated_at, + ), + ) + cur.execute( + "INSERT INTO outbox (job_id, payload) VALUES (%s, %s)", + (outbox_payload["job_id"], _json.dumps(outbox_payload)), + ) + conn.commit() + return attachment + + def claim_processing(self, attachment_id: UUID, lease_seconds: int = 60) -> bool: + with connection() as conn: + with conn.cursor() as cur: + now = now_utc() + cur.execute( + "UPDATE attachments SET status='processing'," + " processing_until=%s, updated_at=%s" + " WHERE id=%s AND status IN ('queued','processing')" + " RETURNING id", + (now + timedelta(seconds=lease_seconds), now, attachment_id), + ) + won = cur.fetchone() is not None + conn.commit() + return won + + def recover_stuck_processing(self) -> int: + with connection() as conn: + with conn.cursor() as cur: + now = now_utc() + cur.execute( + "UPDATE attachments SET status='queued', processing_until=NULL," + " updated_at=%s" + " WHERE status='processing' AND processing_until IS NOT NULL" + " AND processing_until < %s", + (now, now), + ) + count = cur.rowcount + conn.commit() + return count + + def replay_failed_to_queued(self, attachment_id: UUID) -> str: + """Операторский replay: failed → queued с новой записью в outbox. + + Возвращает: + - "requeued" — статус был failed, переведён в queued и запланирован job; + - "already" — запись уже queued/processing/ready (защита готовых), ничего не меняем; + - "missing" — такого вложения нет. + + Ключ эффекта (attachment_id, pipeline_version, variant) и ON CONFLICT + в finalize_ready гарантируют, что повтор не создаст дублей вариантов. + Новый job_id и строка outbox публикуются отдельным процессом outbox, + поэтому разрыв между COMMIT и publish закрыт тем же механизмом, что и при upload. + """ + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT status FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ) + row = cur.fetchone() + if row is None: + conn.rollback() + return "missing" + status = row[0] + if status != "failed": + conn.rollback() + return "already" + + now = now_utc() + cur.execute( + "UPDATE attachments SET status='queued', error_code=NULL," + " processing_until=NULL, updated_at=%s WHERE id=%s", + (now, attachment_id), + ) + + job_id = uuid4() + payload = { + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "created_at": now.isoformat().replace("+00:00", "Z"), + } + cur.execute( + "INSERT INTO outbox (job_id, payload) VALUES (%s, %s)", + (job_id, _json.dumps(payload)), + ) + conn.commit() + return "requeued" + + def finalize_ready( + self, + attachment_id: UUID, + pipeline_version: int, + variants: list[dict], + ) -> None: + with connection() as conn: + with conn.cursor() as cur: + original = next(v for v in variants if v["name"] == "original") + cur.execute( + "UPDATE attachments SET status='ready', error_code=NULL," + " object_key=%s, processing_until=NULL, updated_at=%s" + " WHERE id=%s", + (original["object_key"], now_utc(), attachment_id), + ) + for v in variants: + if v["name"] == "original": + continue + cur.execute( + "INSERT INTO attachment_variants" + " (attachment_id, pipeline_version, variant, object_key," + " content_type, size_bytes, width, height, sha256)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)" + " ON CONFLICT (attachment_id, pipeline_version, variant)" + " DO NOTHING", + ( + attachment_id, pipeline_version, v["name"], + v["object_key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"], + ), + ) + conn.commit() + + def finalize_failed(self, attachment_id: UUID, error_code: str) -> None: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE attachments SET status='failed', error_code=%s," + " processing_until=NULL, updated_at=%s WHERE id=%s", + (error_code, now_utc(), attachment_id), + ) + conn.commit() + + def get_attachment(self, attachment_id: UUID) -> Optional[Attachment]: + pair = self.get_attachment_with_key(attachment_id) + return pair[0] if pair else None + + def get_attachment_with_key( + self, attachment_id: UUID + ) -> Optional[tuple[Attachment, str]]: + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, chat_id, owner_id, filename, content_type, size_bytes," + " sha256, status, error_code, created_at, updated_at, object_key" + " FROM attachments WHERE id = %s", + (attachment_id,), + ) + row = cur.fetchone() + if row is None: + return None + att = _row_to_attachment(row[:11], self.get_attachment_variants(attachment_id)) + return att, row[11] + + def list_attachments_by_ids( + self, ids: List[UUID], chat_id: UUID + ) -> dict[UUID, tuple[Attachment, str]]: + if not ids: + return {} + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, chat_id, owner_id, filename, content_type, size_bytes," + " sha256, status, error_code, created_at, updated_at, object_key" + " FROM attachments WHERE chat_id = %s AND id = ANY(%s)", + (chat_id, list(ids)), + ) + result: dict[UUID, tuple[Attachment, str]] = {} + for row in cur.fetchall(): + att = _row_to_attachment(row[:11]) + result[row[0]] = (att, row[11]) + return result + + def delete_attachment(self, attachment_id: UUID) -> None: + with connection() as conn, conn.cursor() as cur: + cur.execute("DELETE FROM attachments WHERE id = %s", (attachment_id,)) + conn.commit() + + def get_attachment_variants(self, attachment_id: UUID) -> list[dict]: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT variant, content_type, size_bytes, width, height, sha256" + " FROM attachment_variants WHERE attachment_id=%s" + " ORDER BY variant", + (attachment_id,), + ) + return [ + { + "name": r[0], "content_type": r[1], "size_bytes": r[2], + "width": r[3], "height": r[4], "sha256": r[5], + } + for r in cur.fetchall() + ] + + def get_staging_key(self, attachment_id: UUID) -> Optional[str]: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT object_key FROM attachments WHERE id=%s", + (attachment_id,), + ) + row = cur.fetchone() + return row[0] if row else None + + # ---------- Outbox (lab5 level 4) ---------- + def list_unpublished_outbox(self, limit: int = 100) -> list[tuple[int, dict]]: + """Возвращает [(outbox_id, payload_dict), ...]. + + Всегда dict; если payload в БД пришёл строкой (например, из старой + версии), распаковываем через json.loads. + """ + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT id, payload FROM outbox WHERE published_at IS NULL" + " ORDER BY id LIMIT %s", + (limit,), + ) + rows = cur.fetchall() + result: list[tuple[int, dict]] = [] + for r in rows: + raw = r[1] + if isinstance(raw, dict): + payload = raw + else: + payload = _json.loads(raw) + if not isinstance(payload, dict): + # защита от мусора — пропустить + continue + result.append((int(r[0]), payload)) + return result + + def mark_outbox_published(self, outbox_id: int) -> None: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE outbox SET published_at=now() WHERE id=%s", + (outbox_id,), + ) + conn.commit() + + +storage = PostgresStorage() # единственный экземпляр на процесс \ No newline at end of file diff --git a/src/app/storage/rabbit.py b/src/app/storage/rabbit.py new file mode 100644 index 0000000..7fd30d6 --- /dev/null +++ b/src/app/storage/rabbit.py @@ -0,0 +1,155 @@ +"""RabbitMQ: topology, publisher with confirms, consumer с manual ack. + +- Основная очередь: durable quorum queue. +- Retry: TTL-очереди с dead-letter обратно в основную; x-retry-count. +- DLQ: финальные ошибки и исчерпанные retry. +- Publisher confirms: publish() ждёт подтверждения брокера. +- lab6: extra_headers для traceparent/tracestate. +""" +import json +import logging +import os +import threading +from typing import Optional + +import pika + +log = logging.getLogger("rabbit") + +EXCHANGE = "messenger.attachments" +QUEUE = "attachment.process.v1" +RETRY_EXCHANGE = "messenger.attachments.retry" +DLX = "messenger.attachments.dlx" +DLQ = "attachment.process.v1.dlq" + +RETRY_DELAYS = [5, 15, 60] +MAX_ATTEMPTS = len(RETRY_DELAYS) + 1 + +_topology_ready = False +_topology_lock = threading.Lock() + + +def _url() -> str: + return os.environ.get("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/%2F") + + +def connection_params() -> pika.ConnectionParameters: + return pika.URLParameters(_url()) + + +def connect() -> pika.BlockingConnection: + return pika.BlockingConnection(connection_params()) + + +def setup_topology(channel: pika.adapters.blocking_connection.BlockingChannel) -> None: + channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True) + channel.exchange_declare(RETRY_EXCHANGE, exchange_type="direct", durable=True) + channel.exchange_declare(DLX, exchange_type="direct", durable=True) + + channel.queue_declare(DLQ, durable=True, arguments={"x-queue-type": "quorum"}) + channel.queue_bind(DLQ, DLX, routing_key=DLQ) + + channel.queue_declare( + QUEUE, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": DLX, + "x-dead-letter-routing-key": DLQ, + }, + ) + channel.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE) + + for delay in RETRY_DELAYS: + qname = f"attachment.retry.{delay}s" + channel.queue_declare( + qname, + durable=True, + arguments={ + "x-message-ttl": delay * 1000, + "x-dead-letter-exchange": EXCHANGE, + "x-dead-letter-routing-key": QUEUE, + }, + ) + channel.queue_bind(qname, RETRY_EXCHANGE, routing_key=qname) + + +def _ensure_topology(ch: pika.adapters.blocking_connection.BlockingChannel) -> None: + global _topology_ready + if _topology_ready: + return + with _topology_lock: + if _topology_ready: + return + setup_topology(ch) + _topology_ready = True + + +def publish_job(payload: dict, extra_headers: Optional[dict] = None) -> None: + """Публикация job с publisher confirm. extra_headers — traceparent.""" + conn = connect() + try: + ch = conn.channel() + ch.confirm_delivery() + _ensure_topology(ch) + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + headers = dict(extra_headers or {}) + ch.basic_publish( + exchange=EXCHANGE, + routing_key=QUEUE, + body=body.encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + headers=headers or None, + ), + mandatory=True, + ) + finally: + conn.close() + + +def publish_retry(payload: dict, retry_count: int) -> None: + idx = min(retry_count - 1, len(RETRY_DELAYS) - 1) + delay = RETRY_DELAYS[idx] + qname = f"attachment.retry.{delay}s" + conn = connect() + try: + ch = conn.channel() + ch.confirm_delivery() + _ensure_topology(ch) + headers = dict(payload.get("_headers") or {}) + headers["x-retry-count"] = retry_count + clean = {k: v for k, v in payload.items() if k != "_headers"} + body = json.dumps(clean, ensure_ascii=False, separators=(",", ":")) + ch.basic_publish( + exchange=RETRY_EXCHANGE, + routing_key=qname, + body=body.encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + headers=headers, + ), + mandatory=True, + ) + finally: + conn.close() + + +def publish_dlq(payload: dict) -> None: + conn = connect() + try: + ch = conn.channel() + ch.confirm_delivery() + _ensure_topology(ch) + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + ch.basic_publish( + exchange=DLX, + routing_key=DLQ, + body=body.encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=2, content_type="application/json"), + mandatory=True, + ) + finally: + conn.close() \ No newline at end of file diff --git a/src/app/storage/replay_failed.py b/src/app/storage/replay_failed.py new file mode 100644 index 0000000..07527b7 --- /dev/null +++ b/src/app/storage/replay_failed.py @@ -0,0 +1,88 @@ +"""Операторский replay failed-задач: failed → queued + запись в outbox. + +Использование: + python -m app.storage.replay_failed [ ...] + python -m app.storage.replay_failed --all-failed + +Защита: + - переводит ТОЛЬКО failed; queued/processing/ready не трогает (возвращает "already"); + - новая строка outbox с новым job_id — публикация через существующий outbox-процесс; + - идемпотентность эффекта по (attachment_id, pipeline_version, variant) + не даёт создать дубли вариантов, если повторная обработка уже была выполнена. +Логи пишутся только с ID и статусом, без секретов и содержимого файлов. +""" +import logging +import sys +import uuid + +from .db import init_pool, close_pool +from .postgres import storage + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger("replay_failed") + + +def _parse_args(argv): + if not argv: + print( + "usage: python -m app.storage.replay_failed [...] | --all-failed", + file=sys.stderr, + ) + return None + if argv == ["--all-failed"]: + return "all" + ids = [] + for raw in argv: + try: + ids.append(uuid.UUID(raw)) + except ValueError: + print("invalid attachment_id: %r" % raw, file=sys.stderr) + return None + return ids + + +def _list_failed_ids() -> list: + from .db import connection + with connection() as conn, conn.cursor() as cur: + cur.execute("SELECT id FROM attachments WHERE status='failed' ORDER BY updated_at") + return [r[0] for r in cur.fetchall()] + + +def main(argv) -> int: + parsed = _parse_args(argv) + if parsed is None: + return 2 + init_pool() + try: + if parsed == "all": + ids = _list_failed_ids() + else: + ids = parsed + requeued = already = missing = 0 + for aid in ids: + result = storage.replay_failed_to_queued(aid) + if result == "requeued": + requeued += 1 + log.info("event=replay_requeued attachment_id=%s", aid) + elif result == "already": + already += 1 + log.info("event=replay_skipped_not_failed attachment_id=%s", aid) + else: + missing += 1 + log.info("event=replay_missing attachment_id=%s", aid) + log.info( + "event=replay_done requested=%d requeued=%d skipped=%d missing=%d", + len(ids), requeued, already, missing, + ) + return 0 if missing == 0 else 1 + finally: + close_pool() + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/src/app/storage/s3.py b/src/app/storage/s3.py new file mode 100644 index 0000000..a0c9a7c --- /dev/null +++ b/src/app/storage/s3.py @@ -0,0 +1,98 @@ +"""S3-клиент (MinIO) для lab4/lab5. + +Object key генерирует сервер: + staging: staging///original + final: attachments///original + variant: attachments/// +filename используется только для отображения, не в ключе. +""" +import os +from typing import Optional +from uuid import UUID + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError + + +def _endpoint() -> str: + return os.environ["S3_ENDPOINT"] + + +def _bucket() -> str: + return os.environ.get("S3_BUCKET", "messenger-attachments") + + +def client(): + """Клиент с прикладными credentials (не admin). + + Явные короткие таймауты: connect/read — чтобы при недоступности S3 + запросы падали быстро и LB отдавал 503, а не 504 по своему timeout. + """ + return boto3.client( + "s3", + endpoint_url=_endpoint(), + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=os.environ["S3_APP_KEY_ID"], + aws_secret_access_key=os.environ["S3_APP_SECRET_KEY"], + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=3, + retries={"max_attempts": 0}, + ), + ) + + +def staging_key(chat_id: UUID, attachment_id: UUID) -> str: + """Staging — приватный prefix, worker читает отсюда.""" + return "staging/%s/%s/original" % (chat_id, attachment_id) + + +def original_key(chat_id: UUID, attachment_id: UUID) -> str: + """Финальный ключ оригинала. Не зависит от filename.""" + return "attachments/%s/%s/original" % (chat_id, attachment_id) + + +def variant_key(chat_id: UUID, attachment_id: UUID, variant: str) -> str: + return "attachments/%s/%s/%s" % (chat_id, attachment_id, variant) + + +def put_bytes(key: str, data: bytes, content_type: str) -> None: + client().put_object( + Bucket=_bucket(), + Key=key, + Body=data, + ContentType=content_type, + ContentLength=len(data), + ) + + +def get_object(key: str): + """Возвращает (bytes, content_type). Бросает ClientError при отсутствии.""" + resp = client().get_object(Bucket=_bucket(), Key=key) + body = resp["Body"].read() + ctype = resp.get("ContentType") or "application/octet-stream" + return body, ctype + + +def delete_object(key: str) -> None: + client().delete_object(Bucket=_bucket(), Key=key) + + +def head_object(key: str) -> Optional[dict]: + try: + return client().head_object(Bucket=_bucket(), Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"): + return None + raise + + +def list_objects(prefix: str = "attachments/"): + """Итератор по объектам bucket. Используется cleanup-сервисом.""" + paginator = client().get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=_bucket(), Prefix=prefix): + for obj in page.get("Contents", []): + yield obj \ No newline at end of file diff --git a/src/app/storage/s3_cleanup.py b/src/app/storage/s3_cleanup.py new file mode 100644 index 0000000..3d0d4cb --- /dev/null +++ b/src/app/storage/s3_cleanup.py @@ -0,0 +1,176 @@ +"""Фоновая сверка S3 и PostgreSQL (уровень 4 lab4/lab5). + +- Orphan: объект в S3 есть, а метаданных/вариантов в БД нет. + Удаляем только если объект старше ORPHAN_GRACE_SECONDS. +- Missing: метаданные есть, объекта в S3 нет — warning. +- Staging: объекты staging/ старше grace удаляются (не привязаны к текущей работе). +- Также периодически возвращает зависшие processing в queued. +""" +import logging +import os +import sys +import time +from datetime import datetime, timedelta, timezone + +from . import s3 +from .db import connection, init_pool, close_pool +from .postgres import storage + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger("s3_cleanup") + + +def _interval() -> int: + try: + return max(30, int(os.environ.get("CLEANUP_INTERVAL_SECONDS", "300"))) + except ValueError: + return 300 + + +def _grace() -> int: + try: + return max(0, int(os.environ.get("ORPHAN_GRACE_SECONDS", "3600"))) + except ValueError: + return 3600 + + +def _known_keys() -> set[str]: + """Все ожидаемые object key: оригиналы + варианты.""" + keys: set[str] = set() + with connection() as conn, conn.cursor() as cur: + cur.execute("SELECT id, chat_id, object_key FROM attachments") + for aid, cid, okey in cur.fetchall(): + keys.add(okey) + keys.add(s3.original_key(cid, aid)) + cur.execute("SELECT attachment_id, variant FROM attachment_variants") + # нужен chat_id для variant_key + # variants: отдельным запросом с join + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT a.chat_id, v.attachment_id, v.variant" + " FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id" + ) + for cid, aid, variant in cur.fetchall(): + keys.add(s3.variant_key(cid, aid, variant)) + return keys + + +def _active_attachment_ids() -> set[str]: + """ID вложений в статусах queued/processing — их staging трогать нельзя.""" + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id FROM attachments WHERE status IN ('queued', 'processing')" + ) + return {str(r[0]) for r in cur.fetchall()} + + +def _run_once() -> dict: + now = datetime.now(timezone.utc) + grace_cutoff = now - timedelta(seconds=_grace()) + + known = _known_keys() + active_ids = _active_attachment_ids() + orphan_deleted = 0 + missing = 0 + scanned = 0 + staging_deleted = 0 + + # attachments/ — orphan по known + for obj in s3.list_objects("attachments/"): + scanned += 1 + key = obj["Key"] + if key in known: + continue + # Orphan: в S3 есть, в БД нет. Удаляем только старше grace period. + lm = obj.get("LastModified") + if lm is None: + continue + if lm.tzinfo is None: + lm = lm.replace(tzinfo=timezone.utc) + if lm > grace_cutoff: + # Свежий объект — возможно in-flight upload; не трогаем. + continue + try: + s3.delete_object(key) + orphan_deleted += 1 + log.info("event=orphan_deleted key=%s", key) + except Exception as exc: # noqa: BLE001 + log.error("event=orphan_delete_failed key=%s type=%s", + key, type(exc).__name__) + + # staging/ — удаляем только старое И не принадлежащее активной задаче. + # Ключ: staging///original + for obj in s3.list_objects("staging/"): + key = obj["Key"] + parts = key.split("/") + # Ожидаем минимум 4 части: staging / chat_id / attachment_id / original + if len(parts) < 4 or parts[0] != "staging": + continue + attachment_id = parts[2] + if attachment_id in active_ids: + # Задача ещё в queued/processing — staging нужен worker'у. + continue + lm = obj.get("LastModified") + if lm is None: + continue + if lm.tzinfo is None: + lm = lm.replace(tzinfo=timezone.utc) + if lm > grace_cutoff: + continue + try: + s3.delete_object(key) + staging_deleted += 1 + except Exception: + pass + + # missing + for key in known: + if s3.head_object(key) is None: + missing += 1 + log.warning("event=missing_object key=%s", key) + + # зависшие processing → queued + try: + recovered = storage.recover_stuck_processing() + if recovered: + log.info("event=stuck_processing_recovered count=%s", recovered) + except Exception as exc: # noqa: BLE001 + log.error("event=recover_stuck_failed type=%s", type(exc).__name__) + + return { + "scanned": scanned, + "orphan_deleted": orphan_deleted, + "missing": missing, + "known": len(known), + "staging_deleted": staging_deleted, + } + + +def main() -> None: + init_pool() + interval = _interval() + log.info("event=cleanup_start interval_s=%s grace_s=%s", interval, _grace()) + try: + while True: + try: + stats = _run_once() + log.info( + "event=cleanup_pass scanned=%s known=%s orphan_deleted=%s" + " missing=%s staging_deleted=%s", + stats["scanned"], stats["known"], stats["orphan_deleted"], + stats["missing"], stats["staging_deleted"], + ) + except Exception as exc: # noqa: BLE001 + log.error("event=cleanup_failed type=%s", type(exc).__name__) + time.sleep(interval) + finally: + close_pool() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/app/storage/sessions.py b/src/app/storage/sessions.py new file mode 100644 index 0000000..4c368f8 --- /dev/null +++ b/src/app/storage/sessions.py @@ -0,0 +1,143 @@ +"""Хранилище сессий. Redis (основной режим, lab2 level 4) или PostgreSQL (fallback). + +Ключевое: при недоступности Redis поднимаем RuntimeError("redis_unavailable"), +чтобы main.py отдал 503 unavailable, а не 500 internal_error. +""" +import json +import os +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional +from uuid import UUID + +from ..models.schemas import User +from . import db as _db + +try: + import redis as _redis +except ImportError: # pragma: no cover + _redis = None + + +_SESSION_PREFIX = "sess:" + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def _store() -> str: + return os.environ.get("SESSION_STORE", "redis").strip().lower() + + +def _redis_client(): + """Возвращает подключённый клиент Redis или поднимает RuntimeError. + + RuntimeError("redis_unavailable: ...") ловит main.runtime_error_handler → 503. + """ + if _redis is None: + raise RuntimeError("redis_unavailable: redis library is not installed") + url = os.environ.get("REDIS_URL", "redis://redis:6379/0") + try: + client = _redis.from_url( + url, + decode_responses=True, + socket_timeout=2.0, + socket_connect_timeout=2.0, + ) + client.ping() + return client + except _redis.RedisError as exc: + raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc + + +def _user_key(user_id: UUID) -> str: + return str(user_id) + + +def create_session(user_id: UUID, ttl_seconds: int) -> tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = now_utc() + timedelta(seconds=ttl_seconds) + if _store() == "redis": + r = _redis_client() + payload = json.dumps({"user_id": _user_key(user_id), "expires_at": expires_at.isoformat()}) + try: + # SETEX: TTL — сама суть Redis-сессий + r.setex(_SESSION_PREFIX + token, ttl_seconds, payload) + except _redis.RedisError as exc: + raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc + else: + with _db.connection() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO sessions (token, user_id, expires_at) VALUES (%s, %s, %s)", + (token, user_id, expires_at), + ) + conn.commit() + return token, expires_at + + +def get_session_user(token: str) -> Optional[User]: + if _store() == "redis": + r = _redis_client() + try: + raw = r.get(_SESSION_PREFIX + token) + except _redis.RedisError as exc: + raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc + if raw is None: + return None + try: + data = json.loads(raw) + user_id = UUID(data["user_id"]) + expires_at = datetime.fromisoformat(data["expires_at"]) + except Exception: + return None + if expires_at <= now_utc(): + try: + r.delete(_SESSION_PREFIX + token) + except _redis.RedisError: + pass + return None + return _load_user(user_id) + # postgres fallback + with _db.connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token = %s", + (token,), + ) + row = cur.fetchone() + if row is None: + return None + if row[4] <= now_utc(): + cur.execute("DELETE FROM sessions WHERE token = %s", (token,)) + conn.commit() + return None + return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3]) + + +def revoke_session(token: str) -> None: + if _store() == "redis": + r = _redis_client() + try: + r.delete(_SESSION_PREFIX + token) + except _redis.RedisError as exc: + raise RuntimeError("redis_unavailable: %s" % type(exc).__name__) from exc + return + with _db.connection() as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM sessions WHERE token = %s", (token,)) + conn.commit() + + +def _load_user(user_id: UUID) -> Optional[User]: + with _db.connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ) + row = cur.fetchone() + return User(id=row[0], username=row[1], display_name=row[2], created_at=row[3]) if row else None \ No newline at end of file diff --git a/src/app/telemetry.py b/src/app/telemetry.py new file mode 100644 index 0000000..01efa89 --- /dev/null +++ b/src/app/telemetry.py @@ -0,0 +1,160 @@ +"""Единая точка инициализации OpenTelemetry для API и worker. + +- OTLP/HTTP → Collector (otel-collector:4318). +- resource: service.name/version/instance.id/deployment.environment.name=lab. +- LoggingFilter подставляет trace_id/span_id в каждую запись, чтобы они + попадали и в stdout (видно глазами), и в Loki (в тексте строки) — это + требование lab6 level4 (связь лога и trace). +""" +import logging +import os +import uuid + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + + +_INITIALIZED = False +INSTANCE_ID = os.environ.get("INSTANCE_ID") or uuid.uuid4().hex[:12] + + +def _endpoint() -> str: + return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + + +def _resource(service_name: str, service_version: str) -> Resource: + return Resource.create( + { + "service.name": service_name, + "service.version": service_version, + "service.instance.id": INSTANCE_ID, + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + } + ) + + +def _build_trace_provider(resource: Resource) -> TracerProvider: + provider = TracerProvider(resource=resource) + exporter = OTLPSpanExporter(endpoint=_endpoint() + "/v1/traces", timeout=5) + provider.add_span_processor( + BatchSpanProcessor(exporter, max_queue_size=2048, + max_export_batch_size=512, + schedule_delay_millis=2000) + ) + return provider + + +def _build_meter_provider(resource: Resource) -> MeterProvider: + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=_endpoint() + "/v1/metrics", timeout=5), + export_interval_millis=5000, + ) + return MeterProvider(resource=resource, metric_readers=[reader]) + + +def _build_logger_provider(resource: Resource) -> LoggerProvider: + provider = LoggerProvider(resource=resource) + provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=_endpoint() + "/v1/logs", timeout=5), + max_queue_size=2048, + max_export_batch_size=512, + schedule_delay_millis=2000, + ) + ) + return provider + + +class TraceContextFilter(logging.Filter): + """Подставляет record.trace_id / record.span_id из активного span. + + Если активного span нет — пишет "0"*32 и "0"*16, чтобы лог-формат + оставался валидным (не падал с KeyError). Так же OTel сам поступает. + """ + + def filter(self, record: logging.LogRecord) -> bool: + span = trace.get_current_span() + span_ctx = span.get_span_context() if span else None + if span_ctx is not None and span_ctx.is_valid: + record.trace_id = format(span_ctx.trace_id, "032x") + record.span_id = format(span_ctx.span_id, "016x") + else: + record.trace_id = "0" * 32 + record.span_id = "0" * 16 + return True + + +_LOG_FORMAT = ( + "%(asctime)s %(levelname)s %(name)s " + "trace_id=%(trace_id)s span_id=%(span_id)s %(message)s" +) + + +def install_log_format() -> None: + """Единый формат логов API/worker/outbox. + + Поля trace_id/span_id гарантированно существуют благодаря + TraceContextFilter. Их наличие обеспечивает уровень 4 (корреляция + логов и trace) и делает Loki-запросы `|= "trace_id="` рабочими. + """ + handler = logging.StreamHandler() + handler.addFilter(TraceContextFilter()) + handler.setFormatter(logging.Formatter(_LOG_FORMAT)) + root = logging.getLogger() + root.handlers = [handler] + root.setLevel(logging.INFO) + + +def init_telemetry(service_name: str, service_version: str = "1.6.0") -> None: + """Идемпотентная инициализация OTel для процесса.""" + global _INITIALIZED + if _INITIALIZED: + return + + # Сначала — единый формат логов и фильтр trace_id. + install_log_format() + + resource = _resource(service_name, service_version) + trace.set_tracer_provider(_build_trace_provider(resource)) + metrics.set_meter_provider(_build_meter_provider(resource)) + logger_provider = _build_logger_provider(resource) + set_logger_provider(logger_provider) + + # OTel handler шлёт те же записи в Collector → Loki. Он тоже видит + # trace_id из record, потому что span-контекст един. + # + # ВАЖНО: LoggingHandler.emit() формирует тело через self.format(record), + # где self.formatter наследуется от logging.Handler. Без явного + # formatter'а тело будет голым сообщением (msg % args) — без + # trace_id=/span_id=. Это ломало поиск в Loki `|= "trace_id="`. + # Поэтому задаём тот же формат и навешиваем TraceContextFilter + # непосредственно на OTel-хэндлер: фильтр должен отработать ДО emit, + # чтобы record.trace_id существовал на момент форматирования. + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.addFilter(TraceContextFilter()) + otel_handler.setFormatter(logging.Formatter(_LOG_FORMAT)) + logging.getLogger().addHandler(otel_handler) + + _INITIALIZED = True + logging.getLogger("telemetry").info( + "event=otel_init service=%s instance_id=%s endpoint=%s", + service_name, INSTANCE_ID, _endpoint(), + ) + + +def tracer(name: str = "app"): + return trace.get_tracer(name) + + +def meter(name: str = "app"): + return metrics.get_meter(name) \ No newline at end of file diff --git a/src/app/worker.py b/src/app/worker.py new file mode 100644 index 0000000..2528eb8 --- /dev/null +++ b/src/app/worker.py @@ -0,0 +1,405 @@ +"""Worker lab5/lab6: обработка вложений из RabbitMQ с трейсингом. + +- at-least-once, идемпотентно, retry через TTL-очередь, DLQ. +- OTel: consumer span на каждый job, извлечение trace context из заголовков + сообщения (W3C traceparent/tracestate), child spans на S3/DB операции. +- Метрики: jobs_processed_total{outcome}, job_processing_duration_seconds. +""" +import io +import json +import logging +import os +import signal +import sys +import time + +import pika +from PIL import Image, ImageOps +from opentelemetry import context as otel_context, trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from .storage import s3 +from .storage.db import init_pool, close_pool +from .storage.postgres import storage +from .storage.rabbit import ( + QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology, + publish_retry, publish_dlq, +) +from .telemetry import init_telemetry, meter + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) +log = logging.getLogger("worker") + +PIPELINE_VERSION = 1 +LEASE_SECONDS = 60 +THUMB_SIZE = (256, 256) +CROP_SIZE = (128, 128) +OPT_SIZE = (1280, 1280) +OPT_QUALITY = 82 + +_stop = False +_propagator = TraceContextTextMapPropagator() + + +# OTel инициализация — до первого сообщения +init_telemetry("messenger-worker", "1.6.0") +_tracer = trace.get_tracer("worker") +_meter = meter("messenger-worker") +_jobs_counter = _meter.create_counter( + "jobs_processed_total", description="Jobs processed by outcome", unit="1", +) +# Явные границы бакетов в СЕКУНДАХ: job-обработка (decode + resize + 3×S3 PUT +# + DB) обычно от десятков мс до единиц секунд. Дефолтные экспоненциальные +# бакеты слишком грубы для p95/p99 на этой шкале. +_JOB_DURATION_BUCKETS = [ + 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5, + 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0, +] +_jobs_hist = _meter.create_histogram( + "job_processing_duration_seconds", + description="Job processing duration", + unit="s", + explicit_bucket_boundaries_advisory=_JOB_DURATION_BUCKETS, +) + + +# --- span helpers: дочерние операции S3/DB --- +# +# ВАЖНО: атрибуты, содержащие точку (`db.operation`, `aws.s3.key` и т.п.), +# нельзя передавать как kwargs — Python-идентификатор с точкой невозможен. +# Поэтому оба хелпера принимают dict атрибутов. + +def _s3_span(name: str, attrs: dict): + """Context manager для S3-операции (CLIENT span внутри job.consume).""" + return _tracer.start_as_current_span( + name, + kind=SpanKind.CLIENT, + attributes={"messaging.system": "s3", **attrs}, + ) + + +def _db_span(name: str, attrs: dict): + """Context manager для DB-операции (CLIENT span внутри job.consume).""" + return _tracer.start_as_current_span( + name, + kind=SpanKind.CLIENT, + attributes={"db.system": "postgresql", **attrs}, + ) + + +def _handle_sigterm(signum, frame): + global _stop + log.info("event=sigterm_received signal=%s", signum) + _stop = True + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _to_rgb_white(img: Image.Image) -> Image.Image: + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA", "P"): + img = img.convert("RGBA") + bg = Image.new("RGB", img.size, (255, 255, 255)) + bg.paste(img, mask=img.split()[-1]) + return bg + if img.mode != "RGB": + return img.convert("RGB") + return img + + +def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]: + w, h = img.size + scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0) + nw = max(1, int(w * scale)) + nh = max(1, int(h * scale)) + thumb = img.resize((nw, nh), Image.LANCZOS) + buf = io.BytesIO() + thumb.save(buf, format="JPEG", quality=88, optimize=True) + return thumb, buf.getvalue() + + +def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]: + w, h = img.size + side = min(w, h) + left = (w - side) // 2 + top = (h - side) // 2 + sq = img.crop((left, top, left + side, top + side)) + crop = sq.resize(CROP_SIZE, Image.LANCZOS) + buf = io.BytesIO() + crop.save(buf, format="JPEG", quality=88, optimize=True) + return crop, buf.getvalue() + + +def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]: + w, h = img.size + scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0) + nw = max(1, int(w * scale)) + nh = max(1, int(h * scale)) + opt = img.resize((nw, nh), Image.LANCZOS) + buf = io.BytesIO() + opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4) + return opt, buf.getvalue() + + +def _sha256(data: bytes) -> str: + import hashlib + return hashlib.sha256(data).hexdigest() + + +def _s3_put_variant(chat_id, att_id, name: str, data: bytes, ctype: str) -> str: + """Общий путь: S3 PUT варианта внутри CLIENT span.""" + key = s3.variant_key(chat_id, att_id, name) + with _s3_span("s3.put_object", { + "aws.s3.key": key, + "s3.operation": "put_object", + "s3.variant": name, + "s3.content_type": ctype, + "s3.size_bytes": len(data), + }): + s3.put_bytes(key, data, ctype) + return key + + +def process_job(payload: dict) -> None: + """Обработка одного job. Идемпотентна через finalize_ready. + + Внутри активного job.consume (CONSUMER) span создаются дочерние CLIENT + spans для S3 и DB, чтобы разложить задержку обработки по зависимостям. + """ + attachment_id = payload["attachment_id"] + pipeline_version = payload.get("pipeline_version", 1) + job_id = payload["job_id"] + + log.info("event=job_received job_id=%s attachment_id=%s attempt=%s", + job_id, attachment_id, payload.get("attempt", 1)) + + with _db_span("db.claim_processing", { + "db.operation": "UPDATE", + "attachment_id": str(attachment_id), + }): + claimed = storage.claim_processing(attachment_id, LEASE_SECONDS) + if not claimed: + log.info("event=job_skipped_already_done attachment_id=%s", attachment_id) + return + + pair = storage.get_attachment_with_key(attachment_id) + if pair is None: + log.warning("event=attachment_missing attachment_id=%s", attachment_id) + return + att, staging_key = pair + if att.status == "ready": + log.info("event=already_ready attachment_id=%s", attachment_id) + return + + try: + with _s3_span("s3.get_object", { + "aws.s3.key": staging_key, + "s3.operation": "get_object", + "s3.variant": "staging", + }): + raw, ctype = s3.get_object(staging_key) + except Exception as exc: # noqa: BLE001 + raise RuntimeError("staging_unavailable") from exc + + try: + img = Image.open(io.BytesIO(raw)) + img.load() + except Exception as exc: # noqa: BLE001 + with _db_span("db.finalize_failed", { + "db.operation": "UPDATE", + "error_code": "invalid_image", + }): + storage.finalize_failed(attachment_id, "invalid_image") + log.warning("event=invalid_image attachment_id=%s type=%s", + attachment_id, type(exc).__name__) + return + + img_rgb = _to_rgb_white(img) + + # Финал: S3 PUT оригинала + final_key = s3.original_key(att.chat_id, att.id) + with _s3_span("s3.put_object", { + "aws.s3.key": final_key, + "s3.operation": "put_object", + "s3.variant": "original", + "s3.content_type": att.content_type, + "s3.size_bytes": len(raw), + }): + s3.put_bytes(final_key, raw, att.content_type) + + variants = [{ + "name": "original", + "object_key": final_key, + "content_type": att.content_type, + "size_bytes": len(raw), + "width": img.width, + "height": img.height, + "sha256": _sha256(raw), + }] + + # Thumbnail + _, thumb_bytes = _make_thumbnail(img_rgb) + thumb_key = _s3_put_variant(att.chat_id, att.id, "thumbnail", thumb_bytes, "image/jpeg") + tw, th = Image.open(io.BytesIO(thumb_bytes)).size + variants.append({ + "name": "thumbnail", "object_key": thumb_key, + "content_type": "image/jpeg", "size_bytes": len(thumb_bytes), + "width": tw, "height": th, "sha256": _sha256(thumb_bytes), + }) + + # Crop + _, crop_bytes = _make_crop(img_rgb) + crop_key = _s3_put_variant(att.chat_id, att.id, "crop", crop_bytes, "image/jpeg") + cw, ch = Image.open(io.BytesIO(crop_bytes)).size + variants.append({ + "name": "crop", "object_key": crop_key, + "content_type": "image/jpeg", "size_bytes": len(crop_bytes), + "width": cw, "height": ch, "sha256": _sha256(crop_bytes), + }) + + # Optimized + _, opt_bytes = _make_optimized(img_rgb) + opt_key = _s3_put_variant(att.chat_id, att.id, "optimized", opt_bytes, "image/webp") + ow, oh = Image.open(io.BytesIO(opt_bytes)).size + variants.append({ + "name": "optimized", "object_key": opt_key, + "content_type": "image/webp", "size_bytes": len(opt_bytes), + "width": ow, "height": oh, "sha256": _sha256(opt_bytes), + }) + + with _db_span("db.finalize_ready", { + "db.operation": "UPDATE", + "pipeline_version": pipeline_version, + "variants_count": len(variants), + }): + storage.finalize_ready(attachment_id, pipeline_version, variants) + + # Cleanup staging — best-effort, не критичен для результата. + try: + with _s3_span("s3.delete_object", { + "aws.s3.key": staging_key, + "s3.operation": "delete_object", + "s3.variant": "staging", + }): + s3.delete_object(staging_key) + except Exception: + pass + + log.info("event=job_ready attachment_id=%s job_id=%s variants=%d", + attachment_id, job_id, len(variants)) + + +def _extract_trace_context(properties) -> otel_context.Context: + """Извлекает W3C traceparent/tracestate из headers. Ошибка → пустой контекст.""" + try: + headers = dict(properties.headers or {}) + carrier = { + k.lower(): str(v) + for k, v in headers.items() + if k.lower() in ("traceparent", "tracestate") + } + if not carrier: + return otel_context.Context() + return _propagator.extract(carrier=carrier) + except Exception as exc: # noqa: BLE001 + log.warning("event=trace_extract_failed type=%s", type(exc).__name__) + return otel_context.Context() + + +def _on_message(ch, method, properties, body): + """Callback consumer. ack/nack — явно. Оборачивает обработку в CONSUMER span.""" + parent_ctx = _extract_trace_context(properties) + started = time.monotonic() + with _tracer.start_as_current_span( + "job.consume", context=parent_ctx, kind=SpanKind.CONSUMER, + attributes={"messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE}, + ) as span: + try: + payload = json.loads(body) + except Exception: + log.error("event=invalid_job_json") + _jobs_counter.add(1, {"job.type": "unknown", "outcome": "invalid"}) + ch.basic_nack(method.delivery_tag, requeue=False) + return + + headers = dict(properties.headers or {}) + retry_count = int(headers.get("x-retry-count", 0)) + payload["attempt"] = retry_count + 1 + payload["_headers"] = headers + span.set_attribute("job.id", str(payload.get("job_id", ""))) + span.set_attribute("job.type", str(payload.get("type", "unknown"))) + span.set_attribute("job.attempt", retry_count + 1) + + try: + process_job(payload) + duration = time.monotonic() - started + _jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "success"}) + _jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "success"}) + span.set_status(Status(StatusCode.OK)) + ch.basic_ack(method.delivery_tag) + except Exception as exc: # noqa: BLE001 + duration = time.monotonic() - started + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + log.warning("event=job_retry attachment_id=%s attempt=%d type=%s", + payload.get("attachment_id"), retry_count + 1, type(exc).__name__) + if retry_count + 1 >= MAX_ATTEMPTS: + try: + with _db_span("db.finalize_failed", { + "db.operation": "UPDATE", + "error_code": "retry_exhausted", + }): + storage.finalize_failed(payload["attachment_id"], "retry_exhausted") + except Exception: + pass + try: + publish_dlq({k: v for k, v in payload.items() if k != "_headers"}) + except Exception: + pass + _jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "failed"}) + _jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "failed"}) + ch.basic_ack(method.delivery_tag) + return + _jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "retry"}) + _jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "retry"}) + try: + publish_retry(payload, retry_count + 1) + ch.basic_ack(method.delivery_tag) + except Exception: + ch.basic_nack(method.delivery_tag, requeue=True) + + +def main() -> None: + signal.signal(signal.SIGTERM, _handle_sigterm) + init_pool() + log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS) + + while not _stop: + try: + conn = connect() + ch = conn.channel() + setup_topology(ch) + ch.basic_qos(prefetch_count=1) + ch.basic_consume(queue=QUEUE, on_message_callback=_on_message) + log.info("event=consumer_started") + + while not _stop and conn.is_open: + conn.process_data_events(time_limit=1) + + ch.stop_consuming() + conn.close() + except Exception as exc: # noqa: BLE001 + log.error("event=worker_loop_error type=%s", type(exc).__name__) + time.sleep(3) + + close_pool() + log.info("event=worker_stop") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/lab1_run_all.py b/student-tests/lab1_run_all.py new file mode 100644 index 0000000..24e929c --- /dev/null +++ b/student-tests/lab1_run_all.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Единая команда запуска собственных тестов. +Запускает: test_negative.py, test_concurrency.py. +Публичный smoke запускается отдельно.""" +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def run_one(script, base_url): + print("=== %s ===" % script.name) + proc = subprocess.run( + [sys.executable, str(script), "--base-url", base_url], + cwd=str(HERE), + ) + if proc.returncode != 0: + print("FAIL: %s" % script.name) + return False + return True + + +def main(): + import argparse + p = argparse.ArgumentParser(description="Lab1 student tests (level 4)") + p.add_argument("--base-url", default="http://localhost:8080") + args = p.parse_args() + + ok = True + for script in ("lab1_test_negative.py", "lab1_test_concurrency.py"): + if not run_one(HERE / script, args.base_url): + ok = False + if not ok: + raise SystemExit(1) + print("PASS: all student tests (level 4) passed") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/lab1_test_concurrency.py b/student-tests/lab1_test_concurrency.py new file mode 100644 index 0000000..f8f90c1 --- /dev/null +++ b/student-tests/lab1_test_concurrency.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Конкурентный тест: 100 сообщений от 10 клиентов -> ровно 100 уникальных. +Проверяет защиту in-memory структур от гонок (RLock).""" +import sys +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run # noqa: E402 + +TOTAL_MESSAGES = 100 +CLIENTS = 10 + + +def main(): + args = arguments("Lab1 concurrency safety") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + + # Каждый «клиент» — свой actor (alice/bob) для реалистичной конкуренции + actors = [alice, bob] + barrier = threading.Barrier(CLIENTS) + + def worker(worker_id): + local = Client(args.base_url, args.ca_file, timeout=15) + actor = actors[worker_id % len(actors)] + barrier.wait() # синхронный старт всех потоков + produced = [] + for i in range(TOTAL_MESSAGES // CLIENTS): + text = "c-%02d-%02d-%s" % (worker_id, i, uuid.uuid4().hex[:8]) + msg = local.message(actor, chat, text)["json"] + produced.append(msg) + return produced + + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + results = list(pool.map(worker, range(CLIENTS))) + + all_ids = [m["id"] for chunk in results for m in chunk] + check(len(all_ids) == TOTAL_MESSAGES, + "expected %d message ids, got %d" % (TOTAL_MESSAGES, len(all_ids))) + check(len(set(all_ids)) == TOTAL_MESSAGES, + "duplicate message ids under concurrency: %d unique" % len(set(all_ids))) + + # Читаем всю историю через пагинацию + collected = [] + cursor = None + from urllib.parse import quote + for _ in range(50): + q = path + "/messages?limit=100" + if cursor: + q += "&cursor=" + quote(cursor, safe="") + page = c.call("GET", q, actor=alice, schema="MessagePage")["json"] + collected.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + check(len(collected) == TOTAL_MESSAGES, + "stored messages: %d, expected %d" % (len(collected), TOTAL_MESSAGES)) + check(len(set(collected)) == TOTAL_MESSAGES, + "stored messages contain duplicates: %d unique" % len(set(collected))) + + print("PASS: lab1 concurrency: %d messages from %d clients, " + "%d unique stored and readable" % (TOTAL_MESSAGES, CLIENTS, len(set(collected)))) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab1_test_load_track1.py b/student-tests/lab1_test_load_track1.py new file mode 100644 index 0000000..7256a8b --- /dev/null +++ b/student-tests/lab1_test_load_track1.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Трек 1 (оценка 5). Предел in-memory архитектуры. + +Гипотеза: на больших историях список сообщений при чтении каждый раз +копируется и сортируется под RLock, что ухудшает p95/p99 при росте +истории. Убираем сортировку на чтении (append сохраняет порядок) и +сравниваем "до/после" при тех же лимитах. + +Эксперимент: >=3 размера истории (H) x >=3 уровня конкуренции (C). +Метрики: successful RPS, доля ошибок, p50/p95/p99 только успешных +запросов, RSS процесса API (читается через docker stats или /proc). + +Скрипт не меняет API; использует те же публичные клиенты из tests/. +Stdlib only, Python 3.9+. +""" +import json +import math +import statistics +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +# --- профиль эксперимента ------------------------------------------------- +HISTORY_SIZES = [100, 1000, 5000] # >=3 размера истории +CONCURRENCY_LEVELS = [1, 4, 16] # >=3 уровня конкуренции +DURATION_SECONDS = 10 # на каждую точку (H, C) +API_CONTAINER = "mtusi-messenger-lab1" # имя контейнера для docker stats + + +def rss_mb(container: str): + """RSS процесса API в МБ через docker stats --no-stream.""" + try: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container], + capture_output=True, text=True, timeout=10, + ) + if out.returncode != 0: + return None + # Пример: "120.5MiB / 1GiB" + used = out.stdout.strip().split("/")[0].strip() + if used.endswith("MiB"): + return round(float(used[:-3]), 2) + if used.endswith("GiB"): + return round(float(used[:-3]) * 1024, 2) + if used.endswith("KiB"): + return round(float(used[:-3]) / 1024, 2) + except Exception: + return None + return None + + +def seed_history(c, actor, chat, count, workers=16): + """Заполняет историю до count сообщений параллельно.""" + from concurrent.futures import ThreadPoolExecutor + def one(i): + local = Client(c.base, timeout=15) + local.message(actor, chat, "seed-%06d-%s" % (i, uuid.uuid4().hex[:6])) + with ThreadPoolExecutor(max_workers=workers) as pool: + list(pool.map(one, range(count))) + + +def measure_read_point(c, chat, actor, duration, concurrency): + """Закрытый контур: параллельно читаем GET /messages?limit=100.""" + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + barrier = threading.Barrier(concurrency) + latencies, errors = [], 0 + errors_lock = threading.Lock() + + def worker(_): + nonlocal errors + local = Client(c.base, timeout=15) + barrier.wait() + deadline = time.monotonic() + duration + local_lat = [] + while time.monotonic() < deadline: + start = time.monotonic() + try: + local.call("GET", path, actor=actor, schema="MessagePage") + local_lat.append(time.monotonic() - start) + except Failure: + with errors_lock: + errors += 1 + return local_lat + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + results = list(pool.map(worker, range(concurrency))) + elapsed = time.monotonic() - started + + for chunk in results: + latencies.extend(chunk) + latencies.sort() + + def pct(p): + if not latencies: + return None + return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3) + + total = len(latencies) + errors + return { + "history_size": None, # заполнит вызывающий + "concurrency": concurrency, + "duration_s": round(elapsed, 3), + "successful": len(latencies), + "errors": errors, + "error_ratio": round(errors / total, 6) if total else None, + "rps": round(len(latencies) / elapsed, 3), + "latency_ms": {"p50": pct(.50), "p95": pct(.95), "p99": pct(.99)}, + } + + +def main(): + args = arguments("Трек 1: предел in-memory архитектуры") + c = Client(args.base_url, args.ca_file) + + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + + report = {"lab": 1, "track": 1, "profile": { + "history_sizes": HISTORY_SIZES, + "concurrency_levels": CONCURRENCY_LEVELS, + "duration_seconds": DURATION_SECONDS, + "note": "closed-loop; клиент ограничивает throughput; RSS — docker stats", + }, "points": []} + + for h in HISTORY_SIZES: + # Досеиваем историю до h (bootstrap уже создал часть). + # Считаем число сообщений в истории. + # Предыдущая версия: limit=1 - возвращает только одну страницу и далее считает её длину страницы за длину всей истории, + # Текущая версия: limit=100 - делаем полный обход истории по курсору => реальное число сообщений + have = 0 + cursor = None + from urllib.parse import quote + while True: + q = path + "?limit=100" + if cursor: + q += "&cursor=" + quote(cursor, safe="") + page = c.call("GET", q, actor=alice, schema="MessagePage")["json"] + have += len(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + if have < h: + seed_history(c, alice, chat, h - have) + + for con in CONCURRENCY_LEVELS: + point = measure_read_point(c, chat, alice, DURATION_SECONDS, con) + point["history_size"] = h + point["rss_mb"] = rss_mb(API_CONTAINER) + report["points"].append(point) + print(json.dumps(point, ensure_ascii=False)) + + out = Path("evidence/lab1_track1_load.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + print("PASS: track1 load matrix saved to", out) + + +if __name__ == "__main__": + # Дополнительный CLI: --container для имени Docker-контейнера. + import argparse + p = argparse.ArgumentParser(add_help=False) + p.add_argument("--container", default=API_CONTAINER) + extra, _ = p.parse_known_args() + # Прокидываем в namespace аргументов arguments() через env-подобный трюк. + sys.argv = [a for a in sys.argv if not a.startswith("--container")] + import os + os.environ.setdefault("TRACK1_CONTAINER", extra.container) + + def _bind_container(ns): + ns.container = os.environ.get("TRACK1_CONTAINER", API_CONTAINER) + return ns + + # Патчим arguments, чтобы добавить --container. + import client as _client # noqa: E402 + _orig_arguments = _client.arguments + + def _arguments(description, configure=None): + def cfg(parser): + if configure: + configure(parser) + parser.add_argument("--container", default=API_CONTAINER) + return _orig_arguments(description, cfg) + + _client.arguments = _arguments + run(main) \ No newline at end of file diff --git a/student-tests/lab1_test_negative.py b/student-tests/lab1_test_negative.py new file mode 100644 index 0000000..2564c3f --- /dev/null +++ b/student-tests/lab1_test_negative.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Негативные сценарии lab1: неверный JSON, чужой чат, пустое сообщение, +дубликат username, обход нескольких страниц. Stdlib only, Python 3.9+.""" +import sys +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +def _raw_call(base_url, method, path, body=None, headers=None): + """Сырой запрос для проверки malformed JSON (client.py всегда сериализует).""" + url = base_url.rstrip("/") + path + data = body if isinstance(body, (bytes, type(None))) else body.encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def main(): + args = arguments("Lab1 negative scenarios") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + + # 1. Дубликат username -> 409 + c.call("POST", "/api/v1/users", 409, body=alice["registration"]) + + # 2. Неверный JSON (malformed) -> 400 + status, _ = _raw_call( + c.base, + "POST", + "/api/v1/users", + body=b"{not json at all", + headers={"Content-Type": "application/json"}, + ) + check(status == 400, "malformed JSON must be 400, got %s" % status) + + # 3. Лишние поля -> 400 + c.call( + "POST", + "/api/v1/users", + 400, + body={"username": "extra_user_x", "display_name": "X", "unexpected": True}, + ) + + # 4. Неверный тип -> 400 + c.call("POST", "/api/v1/users", 400, body={"username": 123, "display_name": "X"}) + + # 5. Пустое / пробельное сообщение -> 400 + c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}) + c.call("POST", path + "/messages", 400, actor=alice, body={"text": " "}) + + # 6. Чужой чат -> 404 (не 403) + c.call("GET", path, 404, actor=eve) + c.call("GET", path + "/messages", 404, actor=eve) + c.call("POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}) + + # 7. Пагинация: обход нескольких страниц без потерь и дублей + texts = ["p-%02d" % i for i in range(7)] + for t in texts: + c.message(alice, chat, t) + collected = [] + cursor = None + seen_cursors = set() + for _ in range(20): # предохранитель от бесконечного цикла + q = path + "/messages?limit=2" + if cursor: + from urllib.parse import quote + q += "&cursor=" + quote(cursor, safe="") + page = c.call("GET", q, actor=bob, schema="MessagePage")["json"] + collected.extend(m["text"] for m in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + check(cursor not in seen_cursors, "cursor loop detected") + seen_cursors.add(cursor) + check(len(collected) == len(texts), "pagination lost messages: %d != %d" % (len(collected), len(texts))) + check(len(set(collected)) == len(collected), "pagination duplicated messages") + check(collected == texts, "pagination order/contents mismatch") + + # 8. Некорректный limit -> 400 + c.call("GET", path + "/messages?limit=0", 400, actor=alice) + c.call("GET", path + "/messages?limit=101", 400, actor=alice) + + # 9. Неизвестный X-User-Id -> 401 + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": str(uuid.uuid4())}) + + print("PASS: lab1 negative scenarios (dup, malformed JSON, extra fields, " + "wrong type, empty text, foreign chat, pagination, bad limit, unknown user)") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab2_test_level4.py b/student-tests/lab2_test_level4.py new file mode 100644 index 0000000..65c8692 --- /dev/null +++ b/student-tests/lab2_test_level4.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +"""Уровень 4 lab2: expiry TTL, конкурентная регистрация, EXPLAIN, недоступность Redis/PostgreSQL. + +Перед запуском: + 1) в .env выставить SESSION_TTL_SECONDS=5 + 2) docker compose --env-file .env up -d --force-recreate api + 3) docker exec mtuci-redis redis-cli flushdb + 4) python student-tests/test_lab2_level4.py --base-url http://localhost:8080 + 5) перед smoke/persistence вернуть SESSION_TTL_SECONDS=3600 и снова up -d --force-recreate api + +Порядок тестов: тяжёлые по данным и конкурентные — ДО expiry-теста, +потому что при коротком TTL любой долгий шаг съедает окно жизни сессии. +Expiry-тест запускается последним и использует свежие сессии. + +test_explain не зависит от TTL вообще: он сидирует данные напрямую +через psql и проверяет план SQL-запроса, а не HTTP-слой. + +test_chat_rollback проверяет транзакционную границу create_chat: он +запускает патч-скрипт ВНУТРИ контейнера API (там есть psycopg и +DATABASE_URL), monkey-patch-ит psycopg.Cursor.executemany так, чтобы +INSERT в chat_members упал ПОСЛЕ успешного INSERT в chats. Затем через +psql проверяется, что ни chat, ни chat_members не сохранились. +""" +import base64 +import datetime as dt +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + +# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name). +COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default") + +# Имя контейнера API (зафиксировано в compose.yaml container_name). +API_CONTAINER = os.environ.get("API_CONTAINER", "mtuci-messenger-lab2") + + +# ---------- 1. Конкурентная регистрация одинакового username ---------- +def test_concurrent_registration(c): + """Один и тот же username с 8 потоков — ровно одна 201, остальные 409. + + Проверяем ФАКТИЧЕСКИЕ HTTP-коды, а не факт исключения: timeout, 500 + или неверный формат ответа должны проваливать тест, а не выдаваться + за ожидаемый конфликт. Дополнительно сверяем, что в БД создана + ровно одна строка пользователя. + """ + username = "race_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + body = {"username": username, "display_name": "Race", "password": password} + results = [] + lock = threading.Lock() + + def one(_): + local = Client(c.base, timeout=10) + try: + r = local.call("POST", "/api/v1/users", (201, 409), + body=body, schema="User" if True else None) + # schema валидируется только для 2xx; при 409 приходит Error. + status = r["status"] + except Failure as exc: + # Любая иная ошибка (timeout, 500, bad schema) — это не 409. + status = "failure:%s" % exc + with lock: + results.append(status) + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(one, range(8))) + + ok = [r for r in results if r == 201] + conflict = [r for r in results if r == 409] + bad = [r for r in results if r not in (201, 409)] + check(not bad, "unexpected results under concurrent registration: %s" % results) + check(len(ok) == 1, "expected exactly one 201, got %s" % results) + check(len(conflict) == 7, "expected seven 409, got %s" % results) + + # В БД должна быть ровно одна строка с этим username. + dsn = _read_dsn() + rows = _scalar(dsn, "SELECT count(*) FROM users WHERE username = '%s';" % username) + check(int(rows) == 1, + "ожидалась ровно одна строка users для %s, получено %s" % (username, rows)) + + +# ---------- 2. EXPLAIN на реальной БД ---------- +def _read_dsn() -> str: + dsn = os.environ.get("DATABASE_URL") + if not dsn: + env_file = Path(__file__).resolve().parents[1] / ".env" + if env_file.is_file(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith("DATABASE_URL="): + dsn = line.split("=", 1)[1].strip() + break + check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env") + # Postgres доступен только внутри compose-сети; одноразовые psql-контейнеры + # запускаются в этой же сети, поэтому имя хоста postgres резолвится как есть. + return dsn + + +def _psql(dsn_in_container: str, sql: str) -> str: + r = subprocess.run( + [ + "docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "postgres:18-alpine", + "psql", dsn_in_container, "-tA", "-c", sql, + ], + capture_output=True, text=True, timeout=120, + ) + return r.stdout + r.stderr + + +def _scalar(dsn: str, sql: str) -> str: + r = subprocess.run( + [ + "docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "postgres:18-alpine", + "psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1", "-c", sql, + ], + capture_output=True, text=True, timeout=120, + ) + check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr) + return r.stdout.strip().splitlines()[-1] + + +def test_explain(c): + """EXPLAIN (ANALYZE, BUFFERS) истории сообщений на >=10 000 строк. + + Методичка: индекс истории обоснован запросом и EXPLAIN на >=10 000 + сообщениях. На пустой таблице планировщик выбирает Seq Scan, поэтому + тест сам доводит messages до >=10 000 строк и делает ANALYZE. + + Данные сидируются напрямую через psql, без API: тест не зависит от + SESSION_TTL_SECONDS и скорости scrypt. Это корректно, потому что + EXPLAIN проверяет план SQL-запроса, а не HTTP-слой. + """ + dsn_in_container = _read_dsn() + + # Смотрим, сколько строк уже есть. + have_raw = _psql(dsn_in_container, "SELECT count(*) FROM messages;") + have = int(have_raw.strip().splitlines()[-1]) + print(" в таблице messages сейчас: %d строк" % have) + + target_total = 10_000 + if have < target_total: + need = target_total - have + print(" сидируем %d сообщений напрямую через psql ..." % need) + + # Один чат: берём существующий или создаём минимальный. + # gen_random_uuid() доступен в PG13+ без расширений. + seed_sql = """ +DO $$ +DECLARE + v_user UUID; + v_chat UUID; +BEGIN + SELECT id INTO v_user FROM users LIMIT 1; + IF v_user IS NULL THEN + v_user := gen_random_uuid(); + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed', + 'scrypt$16384$8$1$00$00', now()); + END IF; + + SELECT id INTO v_chat FROM chats LIMIT 1; + IF v_chat IS NULL THEN + v_chat := gen_random_uuid(); + INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'Seed chat', now()); + INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user); + END IF; + + -- Доливаем ровно need строк в этот чат. + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), v_chat, v_user, + 'seed-' || g, now() - (g || ' milliseconds')::interval + FROM generate_series(1, %d) AS g; +END $$; +""" % need + r = subprocess.run( + [ + "docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "postgres:18-alpine", + "psql", dsn_in_container, "-c", seed_sql, + ], + capture_output=True, text=True, timeout=120, + ) + seed_out = r.stdout + r.stderr + check("ERROR" not in seed_out, "сидирование не удалось:\n" + seed_out) + + # ANALYZE обязателен: без него планировщик не знает реальных размеров. + _psql(dsn_in_container, "ANALYZE messages;") + _psql(dsn_in_container, "ANALYZE chat_members;") + _psql(dsn_in_container, "ANALYZE chats;") + print(" сидирование завершено") + + # EXPLAIN по САМОМУ БОЛЬШОМУ чату, чтобы планировщик имел основания + # выбрать индекс, а не Seq Scan. + biggest = _psql( + dsn_in_container, + "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;", + ).strip().splitlines()[-1] + check(biggest and len(biggest) >= 32, "не удалось выбрать самый большой чат: %r" % biggest) + + sql = ( + "EXPLAIN (ANALYZE, BUFFERS) " + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + "WHERE chat_id = '%s' " + "ORDER BY created_at, id LIMIT 51;" % biggest + ) + r = subprocess.run( + [ + "docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "postgres:18-alpine", + "psql", dsn_in_container, "-c", sql, + ], + capture_output=True, text=True, timeout=120, + ) + out = r.stdout + r.stderr + print("EXPLAIN (фрагмент):") + for line in out.splitlines()[:12]: + print(" ", line) + check("Index" in out or "Bitmap Index" in out or "idx_messages_chat_created" in out, + "EXPLAIN не показал использования индекса:\n" + out) + + +# ---------- 3. Rollback составной операции create_chat ---------- +def test_chat_rollback(c): + """Проверяем, что при падении INSERT в chat_members ПОСЛЕ успешного + INSERT в chats не остаётся ни строки чата, ни частичного членства. + + Как добиться ошибки именно на вставке участников, а не на предварительной + проверке unknown_member? Проверка `SELECT id FROM users WHERE id = ANY(...)` + в postgres.py отсекает неизвестных до INSERT. Значит, нужен сбой именно + в executemany. + + Решение: запускаем патч-скрипт ВНУТРИ контейнера API через docker exec — + там уже установлен psycopg и доступен DATABASE_URL. Скрипт monkey-patch-ит + psycopg.Cursor.executemany так, чтобы он бросил исключение, и вызывает + storage.create_chat с двумя реально существующими пользователями. + Транзакция дойдёт до INSERT в chats, упадёт на executemany и должна + откатиться. + + После этого через psql проверяем: + - в таблице chats нет строки с этим title; + - в таблице chat_members нет строк с этим chat_id; + - через API GET /api/v1/chats пользователь не видит этот чат. + """ + dsn = _read_dsn() + + # Готовим двух валидных пользователей через API, чтобы FK был валиден. + suffix = uuid.uuid4().hex[:10] + alice_username = "rb_a_" + suffix + bob_username = "rb_b_" + suffix + alice_password = "T3st-" + uuid.uuid4().hex + bob_password = "T3st-" + uuid.uuid4().hex + alice = c.call("POST", "/api/v1/users", 201, + body={"username": alice_username, "display_name": "RB-A", + "password": alice_password}, + schema="User")["json"] + bob = c.call("POST", "/api/v1/users", 201, + body={"username": bob_username, "display_name": "RB-B", + "password": bob_password}, + schema="User")["json"] + + # Уникальный title, чтобы точно найти строку, если rollback не сработает. + marker_title = "rollback-chat-" + uuid.uuid4().hex + + # Скрипт запускается ВНУТРИ контейнера API: там есть psycopg и DATABASE_URL. + # Патчит executemany так, чтобы он упал после успешного INSERT в chats. + patch_script = r''' +import os, sys, psycopg +from uuid import UUID + +_orig = psycopg.Cursor.executemany +def boom(self, *args, **kwargs): + raise RuntimeError("patched_executemany_failure") +psycopg.Cursor.executemany = boom + +sys.path.insert(0, "/app/src") +from app.storage.postgres import storage + +try: + storage.create_chat(os.environ["RB_TITLE"], + [UUID(os.environ["RB_ALICE"]), UUID(os.environ["RB_BOB"])]) +except RuntimeError as exc: + if "patched_executemany_failure" in str(exc): + print("PATCHED_FAILURE_OK"); sys.exit(0) + raise +except Exception as exc: + print("UNEXPECTED:%s:%s" % (type(exc).__name__, exc)); sys.exit(2) +print("NO_FAILURE"); sys.exit(3) +''' + + proc = subprocess.run( + ["docker", "exec", + "-e", "RB_TITLE=" + marker_title, + "-e", "RB_ALICE=" + alice["id"], + "-e", "RB_BOB=" + bob["id"], + API_CONTAINER, + "python", "-c", patch_script], + capture_output=True, text=True, timeout=60, + ) + out = (proc.stdout + proc.stderr).strip() + print(" rollback-процесс: exit=%d, out=%r" % (proc.returncode, out[:200])) + check("PATCHED_FAILURE_OK" in out, + "не удалось вызвать ошибку именно на executemany: %s" % out) + + # Проверяем в БД: ни chats, ни chat_members с этим маркером не осталось. + chats_count = _scalar( + dsn, + "SELECT count(*) FROM chats WHERE title = '%s';" % marker_title, + ) + check(int(chats_count) == 0, + "rollback не сработал: строка chats с title=%s осталась (%s)" % (marker_title, chats_count)) + + members_count = _scalar( + dsn, + "SELECT count(*) FROM chat_members cm " + "JOIN chats c ON c.id = cm.chat_id " + "WHERE c.title = '%s';" % marker_title, + ) + check(int(members_count) == 0, + "rollback не сработал: остались строки chat_members (%s)" % members_count) + + # Дополнительно: у пользователя нет чата с таким title через API. + cred = base64.b64encode((alice_username + ":" + alice_password).encode()).decode() + session = c.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + cred}, + schema="Session")["json"] + listed = c.call("GET", "/api/v1/chats?limit=100", + headers={"Authorization": "Bearer " + session["token"]}, + schema="ChatPage")["json"] + check(all(item["title"] != marker_title for item in listed["items"]), + "чат с маркером rollback виден в списке чатов пользователя") + + +# ---------- 4. Недоступность Redis (с действующим токеном) ---------- +def _compose(action, service="redis"): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", action, service], + capture_output=True, text=True, timeout=60, + ) + + +def _wait_ready(c, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + return + except Failure: + time.sleep(1) + raise Failure("API не вернулся в ready за %d секунд после восстановления зависимости" % timeout) + + +def test_redis_unavailable(c): + """Останавливаем Redis: /health/ready → 503, действующий токен не работает. + + Важно: используем ЗАРАНЕЕ полученную рабочую сессию, а не заведомо + неправильный Bearer. Иначе 401 мог бы получиться и без обращения к Redis. + После восстановления Redis прежняя сессия снова должна работать. + """ + print(" (проверка недоступности Redis: docker compose stop redis на ~6 секунд)") + + # Готовим рабочую сессию ДО остановки Redis. + suffix = uuid.uuid4().hex[:10] + username = "redis_" + suffix + password = "T3st-" + uuid.uuid4().hex + c.call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Redis", "password": password}, + schema="User") + cred = base64.b64encode((username + ":" + password).encode()).decode() + session = c.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + cred}, + schema="Session")["json"] + token = session["token"] + + # Контроль: до остановки Redis токен рабочий. + me = c.call("GET", "/api/v1/users/me", 200, + headers={"Authorization": "Bearer " + token}, + schema="User")["json"] + check(me["username"] == username, "токен должен принадлежать созданному пользователю") + + stopped = _compose("stop", "redis") + check(stopped.returncode == 0, + "docker compose stop redis не сработал:\n" + (stopped.stdout + stopped.stderr)) + + try: + time.sleep(3) + # /health/ready должен стать 503. + ready = c.call("GET", "/health/ready", 503) + check(ready["status"] == 503, "readiness must be 503 without Redis") + + # Действующий токен при мёртвом Redis не должен давать доступ: + # сервер не может проверить сессию и обязан ответить 503. + r = c.call("GET", "/api/v1/users/me", (401, 503), + headers={"Authorization": "Bearer " + token}) + check(r["status"] == 503, + "запрос с действующим токеном при мёртвом Redis должен быть 503, получено %s" % r["status"]) + finally: + # Только восстановление. Никаких return/break/continue внутри finally. + started = _compose("start", "redis") + check(started.returncode == 0, + "docker compose start redis не сработал:\n" + (started.stdout + started.stderr)) + + # Ждём готовности — УЖЕ ВНЕ finally. + _wait_ready(c, timeout=60) + + # Прежний токен снова должен работать: сессия пережила рестарт Redis (AOF). + me2 = c.call("GET", "/api/v1/users/me", 200, + headers={"Authorization": "Bearer " + token}, + schema="User")["json"] + check(me2["id"] == me["id"], "прежняя сессия должна снова работать после восстановления Redis") + + +# ---------- 5. Недоступность PostgreSQL ---------- +def test_postgres_unavailable(c): + """Останавливаем PostgreSQL: /health/ready → 503, действующий токен → 503. + + Проверяем: + - /health/ready возвращает 503; + - запрос с действующим токеном возвращает 503 в пределах таймаута; + - анонимный запрос не получает доступ; + - после запуска БД прежний токен работает, ранее записанные данные сохранены. + Восстановление БД выполняется в finally, чтобы не оставлять стенд остановленным. + """ + print(" (проверка недоступности PostgreSQL: docker compose stop postgres на ~6 секунд)") + + # Готовим рабочую сессию и сообщение ДО остановки БД. + alice, bob, eve, chat = c.bootstrap() + marker = "pgdown-" + uuid.uuid4().hex + sent = c.message(alice, chat, marker)["json"] + + # Контроль: до остановки всё работает. + c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User") + + stopped = _compose("stop", "postgres") + check(stopped.returncode == 0, + "docker compose stop postgres не сработал:\n" + (stopped.stdout + stopped.stderr)) + + try: + time.sleep(3) + # /health/ready должен стать 503. + ready = c.call("GET", "/health/ready", 503) + check(ready["status"] == 503, "readiness must be 503 without PostgreSQL") + + # Запрос с действующим токеном: 503 в пределах таймаута. + t0 = time.monotonic() + r = c.call("GET", "/api/v1/users/me", 503, actor=alice) + elapsed = time.monotonic() - t0 + check(r["status"] == 503, + "запрос с действующим токеном при мёртвой БД должен быть 503") + # statement_timeout=5000 + пул timeout=5.0: укладываемся в разумный предел. + check(elapsed < 30, "ответ при мёртвой БД занял %.1f c, ожидался в пределах таймаута" % elapsed) + + # Анонимный запрос не получает доступ. + anon = c.call("GET", "/api/v1/users/me", (401, 503)) + check(anon["status"] in (401, 503), + "анонимный запрос при мёртвой БД должен быть 401 или 503, получено %s" % anon["status"]) + finally: + started = _compose("start", "postgres") + check(started.returncode == 0, + "docker compose start postgres не сработал:\n" + (started.stdout + started.stderr)) + + # Ждём готовности — УЖЕ ВНЕ finally. + _wait_ready(c, timeout=60) + + # Прежний токен работает, ранее записанное сообщение сохранено. + c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User") + messages = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=bob, schema="MessagePage")["json"]["items"] + check(any(m["id"] == sent["id"] and m["text"] == marker for m in messages), + "ранее записанное сообщение не найдено после восстановления PostgreSQL") + + +# ---------- 6. Expiry TTL ---------- +def test_short_ttl_expiry(c): + """TTL=5. Проверяем полный цикл: токен валиден -> истекает -> 401. + + Идёт ПОСЛЕДНИМ: при коротком TTL любая долгая подготовка съедает + окно жизни сессии, поэтому соседние тесты работают раньше. + """ + suffix = uuid.uuid4().hex[:10] + username = "ttl_" + suffix + password = "T3st-" + uuid.uuid4().hex + user = c.call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "TTL", "password": password}, + schema="User")["json"] + cred = base64.b64encode((username + ":" + password).encode()).decode() + session = c.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + cred}, + schema="Session")["json"] + token = session["token"] + + # Контроль: сервер выдал короткий TTL, иначе тест бессмыслен. + expires_at = dt.datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00")) + ttl_actual = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(3.0 <= ttl_actual <= 10, + "SESSION_TTL_SECONDS не короткий (получено %.1f c). " + "Поставьте, пожалуйста, SESSION_TTL_SECONDS=5 в .env и перезапустите API (команда: docker compose --env-file .env up -d api)." % ttl_actual) + + # До истечения токен должен работать. + me = c.call("GET", "/api/v1/users/me", 200, + headers={"Authorization": "Bearer " + token}, + schema="User")["json"] + check(me["id"] == user["id"], "token must identify its owner before expiry") + + # Ждём гарантированного истечения TTL + запас. + remaining = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds() + time.sleep(max(remaining, 0.0) + 1.0) + + # Истёкший токен — 401. + c.call("GET", "/api/v1/users/me", 401, + headers={"Authorization": "Bearer " + token}) + + # Второй вход тем же пользователем даёт рабочий токен: истёк именно + # первый токен, а не пользователь/хранилище целиком. + session2 = c.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + cred}, + schema="Session")["json"] + c.call("GET", "/api/v1/users/me", 200, + headers={"Authorization": "Bearer " + session2["token"]}, + schema="User") + + +# ---------- main ---------- +def main(): + args = arguments("Lab2 level 4 checks", configure=_configure) + c = Client(args.base_url, args.ca_file) + + # Порядок и режимы: + # --only-expiry — ТОЛЬКО expiry-тест (запускать при SESSION_TTL_SECONDS=5) + # --skip-expiry — все тяжёлые тесты, без expiry (при SESSION_TTL_SECONDS=3600) + # без флагов — сначала тяжёлые, expiry последним (при SESSION_TTL_SECONDS=5 + # это сработает только если тяжёлые тесты успевают до истечения TTL, + # поэтому на практике используйте --only-expiry/--skip-expiry) + run_expiry = not args.skip_expiry + run_others = not args.only_expiry + + if run_others: + print("=== test_concurrent_registration ===") + test_concurrent_registration(c) + print("PASS: concurrent registration produces exactly one user") + + print("=== test_chat_rollback ===") + test_chat_rollback(c) + print("PASS: create_chat rollback leaves no chat and no partial membership") + + print("=== test_explain ===") + test_explain(c) + print("PASS: EXPLAIN uses idx_messages_chat_created") + + print("=== test_redis_unavailable ===") + test_redis_unavailable(c) + print("PASS: readiness 503 без Redis, действующий токен отвергнут, затем восстановление") + + print("=== test_postgres_unavailable ===") + test_postgres_unavailable(c) + print("PASS: readiness 503 без PostgreSQL, данные и сессия сохранены") + + if run_expiry: + print("=== test_short_ttl_expiry ===") + test_short_ttl_expiry(c) + print("PASS: session valid before TTL, expires after TTL, 401 on expired token") + + print("PASS: all requested lab2 level-4 checks passed") + + +def _configure(p): + p.add_argument("--only-expiry", action="store_true", + help="запустить только expiry-тест (SESSION_TTL_SECONDS=5)") + p.add_argument("--skip-expiry", action="store_true", + help="запустить всё, кроме expiry (SESSION_TTL_SECONDS=3600)") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab2_test_track2.py b/student-tests/lab2_test_track2.py new file mode 100644 index 0000000..06cc28e --- /dev/null +++ b/student-tests/lab2_test_track2.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO. + +Шаги: + 1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql). + 2. Сравниваем 3 варианта индексов на большом и маленьком чате: + - baseline (нет индекса) -> Seq Scan + - (chat_id) -> Bitmap/Index + Sort + - (chat_id, created_at, id) -> Index Scan без Sort (текущий) + Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read. + 3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки, + pg_indexes_size, pg_total_relation_size. + 4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine, + сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений. + 5. Границы потери данных: во время pg_dump через API пишем новые сообщения, + после restore проверяем, что их нет. + +Все psql-вызовы идут через docker run postgres:18-alpine в compose-сети проекта +(--network mtuci2026lab2_default), DSN указывает на хост postgres. +""" +import json +import os +import random +import subprocess +import sys +import time +import uuid +from pathlib import Path +import base64 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +EVIDENCE.mkdir(exist_ok=True) + +SEED_SQL = ROOT / "student-tests" / "track2_seed.sql" + +# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name). +COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default") + + +# ---------- helpers ---------- +def _read_dsn() -> str: + dsn = os.environ.get("DATABASE_URL") + if not dsn: + env_file = ROOT / ".env" + if env_file.is_file(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith("DATABASE_URL="): + dsn = line.split("=", 1)[1].strip() + break + check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env") + # Postgres доступен только внутри compose-сети; одноразовые psql/pg_dump + # контейнеры запускаются в этой же сети, имя хоста postgres резолвится как есть. + return dsn + + +def _psql(dsn: str, sql: str, extra=None): + cmd = [ + "docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "postgres:18-alpine", + "psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1", + ] + if extra: + cmd += extra + cmd += ["-c", sql] + return subprocess.run(cmd, capture_output=True, text=True, timeout=300) + + +def _psql_file(dsn: str, path: Path): + cmd = [ + "docker", "run", "--rm", "-i", + "--network", COMPOSE_NETWORK, + "-v", f"{path}:/seed.sql:ro", + "postgres:18-alpine", + "psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql", + ] + return subprocess.run(cmd, capture_output=True, text=True, timeout=600) + + +def _scalar(dsn: str, sql: str): + r = _psql(dsn, sql) + check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr) + return r.stdout.strip().splitlines()[-1] + + +# ---------- 1. seed ---------- +def step_seed(dsn: str): + print("=== 1. seed ===") + before = int(_scalar(dsn, "SELECT count(*) FROM messages;")) + print(" messages before:", before) + r = _psql_file(dsn, SEED_SQL) + check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr) + after = int(_scalar(dsn, "SELECT count(*) FROM messages;")) + print(" messages after:", after) + check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after) + dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;") + print(" распределение по чатам:\n" + dist.stdout) + return after + + +# ---------- 2. explain ---------- +def _big_and_small(dsn): + big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;") + small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;") + return big, small + + +def _chat_size(dsn: str, chat_id: str) -> int: + return int(_scalar(dsn, "SELECT count(*) FROM messages WHERE chat_id = '%s';" % chat_id)) + + +def _explain(dsn: str, chat_id: str, label: str): + # Берём средний created_at, чтобы keyset реально резал историю + mid = _scalar(dsn, + "SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id)) + sql = ( + "EXPLAIN (ANALYZE, BUFFERS) " + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + "WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') " + "ORDER BY created_at, id LIMIT 51;" % (chat_id, mid) + ) + r = _psql(dsn, sql) + check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr) + out = r.stdout + print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8])) + for line in out.splitlines(): + print(" ", line) + return out + + +def _drop_indexes(dsn): + _psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;") + _psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;") + + +def _format_section(title: str, chat_size: int, plan: str, note: str = "") -> str: + """Собирает один раздел отчёта в формате образца.""" + head = "## %s\n\n" % title + if note: + head += note.rstrip() + "\n\n" + return head + "```\n" + plan.rstrip() + "\n```\n" + + +def step_explain(dsn: str, total_messages: int): + print("=== 2. EXPLAIN 3 варианта ===") + big, small = _big_and_small(dsn) + big_size = _chat_size(dsn, big) + small_size = _chat_size(dsn, small) + + # (1) без индекса + _drop_indexes(dsn) + _psql(dsn, "ANALYZE messages;") + no_index_big = _explain(dsn, big, "no_index/big") + no_index_small = _explain(dsn, small, "no_index/small") + + # (2) только (chat_id) + _psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);") + _psql(dsn, "ANALYZE messages;") + chat_only_big = _explain(dsn, big, "chat_only/big") + chat_only_small = _explain(dsn, small, "chat_only/small") + + # (3) полный рабочий + _psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;") + _psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);") + _psql(dsn, "ANALYZE messages;") + full_big = _explain(dsn, big, "full/big") + full_small = _explain(dsn, small, "full/small") + + # Заголовок отчёта — как в образце + parts = [] + parts.append( + "# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса\n\n" + "Данные: %d сообщений, неравномерные чаты. " + "Запрос: `SELECT ... FROM messages WHERE chat_id = AND (created_at, id) > (<середина>, …) " + "ORDER BY created_at, id LIMIT 51`.\n" + "Большой чат ~%d, маленький чат ~%d.\n" % (total_messages, big_size, small_size) + ) + + parts.append(_format_section( + "1. Без индекса, большой чат", big_size, no_index_big, + "`Parallel Seq Scan` по всей таблице, затем `Sort`.")) + parts.append(_format_section( + "2. Без индекса, маленький чат", small_size, no_index_small, + "Тот же `Parallel Seq Scan` по всей таблице.")) + parts.append(_format_section( + "3. Индекс (chat_id), большой чат", big_size, chat_only_big, + "`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`.")) + parts.append(_format_section( + "4. Индекс (chat_id), маленький чат", small_size, chat_only_small, + "`Index Scan using idx_messages_chat_only` + `Sort`.")) + parts.append(_format_section( + "5. Полный индекс (chat_id, created_at, id), большой чат", big_size, full_big, + "`Index Scan using idx_messages_chat_created`, **без `Sort`**.")) + parts.append(_format_section( + "6. Полный индекс (chat_id, created_at, id), маленький чат", small_size, full_small, + "`Index Scan` без `Sort`.")) + + (EVIDENCE / "lab2_track2_explain.md").write_text( + "\n".join(parts), encoding="utf-8") + + return {"big_chat": big, "small_chat": small} + + +# ---------- 3. write cost ---------- +def _index_sizes(dsn: str): + out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));") + idx, total = out.split("|") + return idx.strip(), total.strip() + + +def step_write_cost(dsn: str): + print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===") + seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;") + seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;") + + results = [] + for name, setup in ( + ("no_index", lambda: _drop_indexes(dsn)), + ("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"), + _psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))), + ("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"), + _psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))), + ): + setup() + # мерить именно вставку 10k + t0 = time.monotonic() + r = _psql(dsn, + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() " + "FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name)) + dt = time.monotonic() - t0 + check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr) + idx, total = _index_sizes(dsn) + results.append({"variant": name, "insert_10000_s": round(dt, 3), + "indexes_size": idx, "total_size": total}) + print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total)) + + (EVIDENCE / "lab2_track2_write_cost.json").write_text( + json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return results + + +# ---------- 4. backup/restore ---------- +def step_backup_restore(dsn: str): + print("=== 4. backup -> restore -> сверка ===") + out_file = EVIDENCE / "lab2_track2_dump.dump" + if out_file.exists(): + out_file.unlink() + + # pg_dump через docker run (использует образ PG 18), в compose-сети + t0 = time.monotonic() + r = subprocess.run( + ["docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "-v", f"{EVIDENCE}:/out", + "postgres:18-alpine", + "pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn], + capture_output=True, text=True, timeout=600, + ) + dump_time = time.monotonic() - t0 + check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr) + size = out_file.stat().st_size + print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time)) + + # новый контейнер как отдельное хранилище — в той же compose-сети, + # порт наружу не публикуем + restore_container = "mtuci-pg-restore" + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + subprocess.run( + ["docker", "run", "-d", "--name", restore_container, + "--network", COMPOSE_NETWORK, + "-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres", + "-e", "POSTGRES_DB=high_load_messenger", + "postgres:18-alpine"], + capture_output=True, text=True, + ) + # ждём готовности + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"], + capture_output=True, text=True) + if c.returncode == 0: + break + time.sleep(1) + else: + raise Failure("restore container не поднялся") + + # pg_restore -c (clean) на пустую БД; restore-контейнер доступен по имени + # в compose-сети + t0 = time.monotonic() + r = subprocess.run( + ["docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "-v", f"{EVIDENCE}:/in", + "postgres:18-alpine", + "pg_restore", "-d", "postgresql://postgres:restore@mtuci-pg-restore:5432/high_load_messenger", + "--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"], + capture_output=True, text=True, timeout=600, + ) + restore_time = time.monotonic() - t0 + print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time)) + check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr) + + # Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера + def q_restore(sql): + c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres", + "-d", "high_load_messenger", "-tA", "-c", sql], + capture_output=True, text=True, timeout=120) + check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr) + return c.stdout.strip().splitlines()[-1] + + orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;")) + rest_total = int(q_restore("SELECT count(*) FROM messages;")) + check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total)) + + orig_dist = _scalar(dsn, + "SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) " + "FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;") + rest_dist = q_restore( + "SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) " + "FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;") + check(orig_dist == rest_dist, "распределение по чатам не совпадает") + + # 100 случайных сообщений + ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;") + ids = ids_raw.split(",") + check(len(ids) == 100, "не удалось выбрать 100 id") + ids_list = ",".join("'%s'" % i for i in ids) + orig_rows = _scalar(dsn, + "SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list) + rest_rows = q_restore( + "SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list) + check(orig_rows == rest_rows, "100 случайных сообщений различаются") + + print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают") + print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024)) + + (EVIDENCE / "lab2_track2_restore.txt").write_text( + "Track 2 restore\n" + "dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n" + "count_original=%d\ncount_restored=%d\n" + "dist_match=true\nrandom100_match=true\n" + % (dump_time, size, restore_time, orig_total, rest_total), + encoding="utf-8") + + # чистим restore-контейнер (volume анонимный) + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3), + "dump_bytes": size, "count": orig_total} + + +# ---------- 5. RPO ---------- +def step_rpo(dsn: str, base_url: str, ca_file): + print("=== 5. RPO: запись во время pg_dump ===") + c = Client(base_url, ca_file) + + # Данные и сессии готовим заранее, до дампа. + alice, bob, eve, chat = c.bootstrap() + + # Контроль: сессия должна быть валидна, иначе 401 — это не RPO. + me = c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User") + check(me["status"] == 200, "сессия должна быть валидна перед RPO-тестом") + + marker = "rpo-" + uuid.uuid4().hex + # Запускаем pg_dump + out_file = EVIDENCE / "lab2_track2_rpo.dump" + if out_file.exists(): + out_file.unlink() + + # pg_dump: тот же приём, что в шаге 4, в compose-сети. + proc = subprocess.Popen( + ["docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "-v", f"{EVIDENCE}:/out", + "postgres:18-alpine", + "pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + time.sleep(1.0) # снапшот уже снят + + # Свежий логин прямо перед записью: не зависим от TTL старой сессии. + cred = base64.b64encode( + (alice["registration"]["username"] + ":" + + alice["registration"]["password"]).encode() + ).decode() + fresh = c.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + cred}, + schema="Session")["json"] + alice_for_write = {"id": alice["id"], "user": alice["user"], "token": fresh["token"]} + + written = c.message(alice_for_write, chat, marker)["json"] + code = proc.wait(timeout=300) + check(code == 0, "pg_dump для RPO упал") + + # Временный restore-контейнер: в compose-сети, без публикации портов. + restore_container = "mtuci-pg-rpo" + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + subprocess.run( + ["docker", "run", "-d", "--name", restore_container, + "--network", COMPOSE_NETWORK, + "-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres", + "-e", "POSTGRES_DB=high_load_messenger", + "postgres:18-alpine"], + capture_output=True, text=True, + ) + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"], + capture_output=True, text=True) + if c2.returncode == 0: + break + time.sleep(1) + else: + raise Failure("restore container для RPO не поднялся") + + # pg_restore идёт в restore-контейнер по имени в compose-сети. + r = subprocess.run( + ["docker", "run", "--rm", + "--network", COMPOSE_NETWORK, + "-v", f"{EVIDENCE}:/in", + "postgres:18-alpine", + "pg_restore", "-d", + "postgresql://postgres:restore@mtuci-pg-rpo:5432/high_load_messenger", + "--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"], + capture_output=True, text=True, timeout=600, + ) + check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr) + + c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres", + "-d", "high_load_messenger", "-tA", "-c", + "SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]], + capture_output=True, text=True) + present = int(c3.stdout.strip().splitlines()[-1]) + check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)") + print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8]) + + (EVIDENCE / "lab2_track2_rpo.txt").write_text( + "Track 2 RPO\n" + "marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n" + % (written["id"], present), + encoding="utf-8") + + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + + +# ---------- main ---------- +def main(): + args = arguments("Track 2: plan, write cost, backup/restore, RPO") + dsn = _read_dsn() + total = step_seed(dsn) + step_explain(dsn, total) + step_write_cost(dsn) + step_backup_restore(dsn) + step_rpo(dsn, args.base_url, args.ca_file) + print("PASS: track 2 (explain + write cost + backup/restore + RPO)") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab2_track2_explain.sql b/student-tests/lab2_track2_explain.sql new file mode 100644 index 0000000..5e28ed8 --- /dev/null +++ b/student-tests/lab2_track2_explain.sql @@ -0,0 +1,10 @@ +-- Track 2: сравнение планов и стоимости записи. +-- Все три варианта прогоняются на одном большом чате (60k) и одном маленьком (~1400). + +-- 0. Определяем два чата +\set ON_ERROR_STOP on + +SELECT id AS big_chat_id +FROM messages GROUP BY id_chat GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1; + +SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1; \ No newline at end of file diff --git a/student-tests/lab2_track2_seed.sql b/student-tests/lab2_track2_seed.sql new file mode 100644 index 0000000..032f86e --- /dev/null +++ b/student-tests/lab2_track2_seed.sql @@ -0,0 +1,81 @@ +-- Track 2: сидирование >=100 000 сообщений с неравномерными чатами. +-- Запускать через: docker run --rm ... psql "$DATABASE_URL" -f track2_seed.sql +-- Идемпотентно по количеству: если сообщений уже >= 100 000, ничего не делает. + +DO $$ +DECLARE + v_user UUID; + v_total INT; + v_have INT; + v_need INT; + v_chat UUID; + rec RECORD; +BEGIN + SELECT count(*) INTO v_have FROM messages; + v_total := 100000; + SELECT count(*) INTO v_have FROM messages; + IF v_have < 100000 THEN + -- меньше 100к: догоняем до 100к + v_need := 100000 - v_have; -- сколько добавлять за прогон + ELSE + v_need := 50000; -- добавляем фиксированные 50к за прогон + END IF; + + -- Гарантируем хотя бы одного пользователя + SELECT id INTO v_user FROM users LIMIT 1; + IF v_user IS NULL THEN + v_user := gen_random_uuid(); + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed', + 'scrypt$16384$8$1$00$00', now()); + END IF; + + -- Создаём 10 чатов, если их меньше (нужны только для распределения) + -- Неравномерность: 60k / 20k / 10k / по ~1400 на остальные 7. + -- Считаем веса, доливаем только v_need строк. + CREATE TEMP TABLE _plan(chat_no INT, weight INT) ON COMMIT DROP; + INSERT INTO _plan VALUES (1, 60), (2, 20), (3, 10), + (4, 2), (5, 2), (6, 1), (7, 1), + (8, 1), (9, 1), (10, 1); + + FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP + -- Находим/создаём chat_id для этого номера: используем title 'seed-chat-' + SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1; + IF v_chat IS NULL THEN + v_chat := gen_random_uuid(); + INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'seed-chat-' || rec.chat_no, now()); + END IF; + IF NOT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = v_chat AND user_id = v_user) THEN + INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user); + END IF; + END LOOP; + + -- Распределяем v_need по весам (целочисленно), последний чат добирает остаток. + DECLARE + v_sum INT; + v_acc INT := 0; + v_alloc INT; + BEGIN + SELECT sum(weight) INTO v_sum FROM _plan; + FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP + IF rec.chat_no = 10 THEN + v_alloc := v_need - v_acc; -- остаток + ELSE + v_alloc := (v_need::numeric * rec.weight / v_sum)::int; + v_acc := v_acc + v_alloc; + END IF; + + SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1; + + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), v_chat, v_user, + 'seed-' || rec.chat_no || '-' || g, + now() - (g || ' milliseconds')::interval + FROM generate_series(1, v_alloc) AS g; + END LOOP; + END; +END $$; + +ANALYZE messages; +ANALYZE chat_members; +ANALYZE chats; \ No newline at end of file diff --git a/student-tests/lab3_test_level4.py b/student-tests/lab3_test_level4.py new file mode 100644 index 0000000..8f1010d --- /dev/null +++ b/student-tests/lab3_test_level4.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Уровень 4 lab3: два X-Instance-Id, идемпотентность под нагрузкой, отказ одной реплики. + +Запуск: + python student-tests/lab3_test_level4.py --base-url http://localhost:8080 + +Требуется рабочий стенд из compose.yaml (LB + 2 реплики + Postgres + Redis). +""" +import base64 +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +# ---------- 1. Два X-Instance-Id через LB ---------- +def test_two_instances(c): + """Один пользователь, серия запросов — видим >=2 разных X-Instance-Id. + + Nginx с keepalive к upstream прибивает клиента к одной реплике на всё + время соединения. Поэтому каждый проб-запрос шлём с Connection: close, + чтобы LB открывал новое upstream-соединение и честно распределял. + """ + alice, bob, eve, chat = c.bootstrap() + seen = set() + for _ in range(60): + r = c.call("GET", "/api/v1/users/me", actor=alice, schema="User", + headers={"Connection": "close"}) + seen.add(r["headers"]["X-Instance-Id"]) + if len(seen) >= 2: + break + check(len(seen) >= 2, "LB не маршрутизирует на разные API-реплики: %s" % seen) + print(" observed instances:", sorted(seen)) + + +# ---------- 2. Идемпотентность: один ключ на двух репликах ---------- +def test_idempotency_across_replicas(c): + """Одновременный POST с одним Idempotency-Key через LB создаёт ровно одно сообщение. + + Для этого параллельно шлём 10 запросов с одним ключом из 10 потоков. + LB распределит их по разным репликам; благодаря общей таблице + idempotency_keys и UNIQUE-индексу, INSERT выигрывает ровно один. + """ + alice, bob, eve, chat = c.bootstrap() + key = "idem-" + uuid.uuid4().hex + path = "/api/v1/chats/" + chat["id"] + "/messages" + body = {"text": "idem-concurrent"} + lock = threading.Lock() + results = [] + + def one(_): + local = Client(c.base, timeout=10) + try: + r = local.call( + "POST", path, (201, 409), + actor=alice, body=body, + headers={"Idempotency-Key": key}, + schema="Message", + ) + with lock: + results.append((r["status"], r["headers"].get("X-Instance-Id"))) + except Failure as exc: + with lock: + results.append(("failure:%s" % exc, None)) + + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(one, range(10))) + + ok = [r for r in results if r[0] == 201] + conflicts = [r for r in results if r[0] == 409] + bad = [r for r in results if r[0] not in (201, 409)] + check(not bad, "unexpected idempotency results: %s" % results) + check(len(ok) >= 1, "no 201 at all") + check(len(ok) + len(conflicts) == 10, "wrong count") + + # Проверяем в БД: ровно одно сообщение с этим текстом в этом чате. + # Через API: читаем всю историю и считаем. + from urllib.parse import quote + items = [] + cursor = None + for _ in range(50): + q = path + "?limit=100" + if cursor: + q += "&cursor=" + quote(cursor, safe="") + page = c.call("GET", q, actor=bob, schema="MessagePage")["json"] + items.extend(m for m in page["items"] if m["text"] == "idem-concurrent") + cursor = page["next_cursor"] + if cursor is None: + break + check(len(items) == 1, "idempotency created %d messages, expected 1" % len(items)) + + # X-Instance-Id: запросы шли на разные реплики (проверяем, что 201 пришёл + # хотя бы с одной, а 409 — с другой; это ожидание не обязательное, но + # полезный сигнал, что LB реально распределяет). + ok_instances = {r[1] for r in ok} + conflict_instances = {r[1] for r in conflicts} + print(" 201 instances:", ok_instances, " 409 instances:", conflict_instances) + print(" PASS: idempotency OK, ровно одно сообщение создано") + + +# ---------- 3. Отказ одной реплики ---------- +def test_stop_one_replica(c): + """Останавливаем одну реплику; вторая продолжает обслуживать запросы. + + Восстановление делаем явно через `docker start` в finally, а ожидание + готовности — ПОСЛЕ try/finally, чтобы не использовать return в finally. + """ + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + + # Список контейнеров сервиса api (compose project name неизвестен, + # ищем по image/label). + ps = subprocess.run( + ["docker", "ps", "--filter", "label=com.docker.compose.service=api", + "--format", "{{.ID}}"], + capture_output=True, text=True, timeout=10, + ) + ids = [x.strip() for x in ps.stdout.splitlines() if x.strip()] + check(len(ids) >= 2, "нужно >=2 контейнера api, найдено %d" % len(ids)) + + target = ids[0] + print(" останавливаем replica container:", target[:12]) + + # Готовим одно подтверждённое сообщение до отказа. + key0 = "before-" + uuid.uuid4().hex + first = c.call("POST", path, 201, actor=alice, body={"text": "before-fail"}, + headers={"Idempotency-Key": key0}, schema="Message")["json"] + + # Останавливаем реплику. + stop = subprocess.run(["docker", "stop", "-t", "5", target], + capture_output=True, text=True, timeout=30) + check(stop.returncode == 0, "docker stop failed: %s" % stop.stderr) + + try: + # Даём LB обнаружить отказ (proxy_next_upstream, max_fails=2). + time.sleep(3) + + # Читаем и пишем — должны попадать во вторую реплику. + deadline = time.monotonic() + 10 + recovered = False + seen_instances = set() + while time.monotonic() < deadline: + try: + r = c.call("POST", path, 201, actor=alice, + body={"text": "after-fail"}, + headers={"Idempotency-Key": "after-" + uuid.uuid4().hex}, + schema="Message") + seen_instances.add(r["headers"].get("X-Instance-Id")) + recovered = True + break + except Failure: + time.sleep(0.5) + check(recovered, "не удалось записать после отказа за 10 секунд") + + # Убеждаемся, что подтверждённое до отказа сообщение доступно. + from urllib.parse import quote + items, cursor = [], None + while True: + q = path + "?limit=100" + if cursor: + q += "&cursor=" + quote(cursor, safe="") + page = c.call("GET", q, actor=bob, schema="MessagePage")["json"] + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + check(any(m["id"] == first["id"] for m in items), + "подтверждённое сообщение потеряно после отказа") + print(" after-fail instances:", seen_instances) + print(" PASS: стенд пережил отказ одной реплики") + finally: + # Только восстановление реплики. Никаких return/break/continue. + subprocess.run(["docker", "start", target], + capture_output=True, text=True, timeout=30) + + # Ждём готовности через LB — УЖЕ ВНЕ finally. + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + # И двух реплик снова должно быть >=2. + seen = set() + for _ in range(30): + seen.add(c.call("GET", "/api/v1/users/me", actor=alice, + schema="User")["headers"]["X-Instance-Id"]) + if len(seen) >= 2: + break + if len(seen) >= 2: + return + except Failure: + pass + time.sleep(1) + raise Failure("стенд не восстановился до двух реплик за 60 секунд") + + +# ---------- main ---------- +def main(): + args = arguments("Lab3 level 4 checks") + c = Client(args.base_url, args.ca_file) + + print("=== test_two_instances ===") + test_two_instances(c) + print("PASS: >=2 X-Instance-Id через LB") + + print("=== test_idempotency_across_replicas ===") + test_idempotency_across_replicas(c) + print("PASS: идемпотентность на общих ключах") + + print("=== test_stop_one_replica ===") + test_stop_one_replica(c) + print("PASS: отказ одной API-реплики обработан") + + print("PASS: all lab3 level-4 checks passed") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab4_test_level4.py b/student-tests/lab4_test_level4.py new file mode 100644 index 0000000..776270c --- /dev/null +++ b/student-tests/lab4_test_level4.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Level 4 для lab4: негативные сценарии, компенсация при отказе БД, параллельные загрузки. + +Запуск: + $env:API_CONTAINER = (docker ps --filter "label=com.docker.compose.service=api" -q | Select-Object -First 1) + python student-tests/lab4_test_level4.py --base-url http://localhost:8080 + +Сценарий компенсации (уровень 4): + Останавливаем PostgreSQL → делаем upload → S3 PUT успешен, INSERT в БД + падает → API удаляет объект из S3 (компенсация) → 503. + После восстановления PG считаем объекты в bucket через mc — количество + не должно вырасти. Это проверяет требование методички: «отказ между + записью S3/БД не оставляет вечных неучтённых объектов». +""" +import os +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = ROOT / "tests" / "fixtures" / "sample.png" +API_CONTAINER = os.environ.get("API_CONTAINER", "") +COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab4_default") +MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920") + + +def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str): + # filename URL-encoded: иначе urllib откажется формировать URL с NUL/CR/LF. + qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote) + url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs + req = urllib.request.Request( + url, data=data, method="POST", + headers={ + "Authorization": "Bearer " + actor["token"], + "Content-Type": ctype, + }, + ) + try: + with c.opener.open(req, timeout=30) as resp: + import json as _json + return resp.code, _json.loads(resp.read()), resp.headers + except urllib.error.HTTPError as e: + import json as _json + body = e.read() + try: + return e.code, _json.loads(body), e.headers + except Exception: + return e.code, None, e.headers + + +def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"): + return _raw_upload(c, actor, chat["id"], filename, data, ctype) + + +# ---------- негативные сценарии ---------- + +def test_oversize(c): + alice, bob, eve, chat = c.bootstrap() + big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10_485_761) + status, body, _h = _upload(c, alice, chat, big) + check(status == 413, "oversize upload must be 413, got %s" % status) + + +def test_corrupted_image(c): + alice, bob, eve, chat = c.bootstrap() + bad = b"\x89PNG\r\n\x1a\n" + b"not-a-real-png" * 100 + status, body, _h = _upload(c, alice, chat, bad) + check(status == 415, "corrupted PNG must be 415, got %s" % status) + + +def test_mime_spoof(c): + alice, bob, eve, chat = c.bootstrap() + status, body, _h = _upload(c, alice, chat, b"hello, not an image" * 10, ctype="image/png") + check(status == 415, "spoofed MIME must be 415, got %s" % status) + + +def test_bad_filename(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + for bad in ("../evil.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", " "): + status, body, _h = _upload(c, alice, chat, data, filename=bad) + check(status == 400, "bad filename %r must be 400, got %s" % (bad, status)) + + +# ---------- изоляция доступа ---------- + +def test_foreign_attachment_and_cross_chat(c): + alice, bob, eve, chat = c.bootstrap() + chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Второй", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat")["json"] + data = SAMPLE.read_bytes() + status, att, _h = _upload(c, alice, chat2, data) + check(status == 201, "upload to chat2 must be 201, got %s" % status) + att_id = att["id"] + + c.call("GET", "/api/v1/attachments/" + att_id, 404, actor=bob) + c.call("GET", "/api/v1/attachments/" + att_id + "/content", 404, actor=bob) + + c.call( + "POST", "/api/v1/chats/" + chat["id"] + "/messages", + 404, actor=bob, + body={"text": "", "attachment_ids": [att_id]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + +def test_attachment_belongs_to_chat(c): + alice, bob, eve, chat = c.bootstrap() + other = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat")["json"] + data = SAMPLE.read_bytes() + status, att, _h = _upload(c, alice, chat, data) + check(status == 201, "upload to chat must be 201") + c.call( + "POST", "/api/v1/chats/" + other["id"] + "/messages", + 404, actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + +# ---------- компенсация: S3 PUT успешен, INSERT в БД падает ---------- + +def _compose(action, service): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", action, service], + capture_output=True, text=True, timeout=90, + ) + + +def _wait_ready(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + return + except Failure: + time.sleep(1) + raise Failure("API не вернулся в ready за %d секунд" % timeout) + + +def _count_bucket_objects() -> int: + env = os.environ.copy() + env_file = ROOT / ".env" + if env_file.is_file(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip()) + + user = env.get("MINIO_ROOT_USER", "minioadmin") + password = env.get("MINIO_ROOT_PASSWORD", "minioadmin") + bucket = env.get("S3_BUCKET", "messenger-attachments") + mc_host = "http://%s:%s@minio:9000" % (user, password) + + # Entrypoint образа mc обычно сам mc; тогда подкоманду зовём без префикса "mc". + ins = subprocess.run( + ["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"], + capture_output=True, text=True, timeout=10, + ) + ep = (ins.stdout or "").strip() + prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"] + + r = subprocess.run( + ["docker", "run", "--rm", "--network", COMPOSE_NETWORK, + "-e", "MC_HOST_local=" + mc_host, MC_IMAGE] + + prefix + + ["ls", "--recursive", "local/" + bucket], + capture_output=True, text=True, timeout=60, + ) + if r.returncode != 0: + # Диагностика при первом запуске — увидим, что именно не так. + print(" mc stderr:", (r.stderr or "").strip()[:300]) + return -1 + return sum(1 for line in r.stdout.splitlines() if line.startswith("[")) + + +def test_compensation_on_db_failure(c): + """S3 PUT прошёл, INSERT в БД упал → объект удаляется из S3, ответ 503. + + Сценарий: останавливаем PostgreSQL. API-реплика сначала делает PUT в S3 + (успешно), затем INSERT в attachments (падает: connection error). + В except-ветке upload_attachment вызывается s3.delete_object — + orphan не остаётся. Ответ — 503 unavailable. + После восстановления PG сверяем количество объектов в bucket. + """ + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + + before = _count_bucket_objects() + check(before >= 0, "не удалось посчитать объекты в bucket (mc)") + + stopped = _compose("stop", "postgres") + check(stopped.returncode == 0, "docker compose stop postgres failed") + time.sleep(2) + try: + status, body, _h = _upload(c, alice, chat, data) + check(status in (500, 503), + "compensation expected 5xx, got %s (body=%r)" % (status, body)) + print(" upload при мёртвой БД вернул %s" % status) + finally: + started = _compose("start", "postgres") + check(started.returncode == 0, "docker compose start postgres failed") + _wait_ready(c, timeout=90) + + after = _count_bucket_objects() + check(after == before, + "orphan-объект остался в bucket: before=%s after=%s" % (before, after)) + print(" компенсация: bucket не вырос (before=%s after=%s)" % (before, after)) + + +# ---------- параллельные загрузки с измерением RSS ---------- + +def test_parallel_uploads_memory(c): + """Уровень 4: ≥10 одновременных загрузок с измерением пика RSS.""" + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + barrier = threading.Barrier(10) + errors = [] + + def one(_): + local = Client(c.base, timeout=60) + barrier.wait() + try: + status, att, _h = _upload(local, alice, chat, data) + if status != 201: + errors.append("status=%s" % status) + except Exception as exc: # noqa: BLE001 + errors.append(type(exc).__name__) + + def rss(): + if not API_CONTAINER: + return None + try: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", API_CONTAINER], + capture_output=True, text=True, timeout=10, + ) + used = out.stdout.strip().split("/")[0].strip() + if used.endswith("MiB"): + return float(used[:-3]) + if used.endswith("GiB"): + return float(used[:-3]) * 1024 + except Exception: + return None + return None + + peak = {"value": 0.0} + stop = threading.Event() + + def sampler(): + while not stop.is_set(): + v = rss() + if v and v > peak["value"]: + peak["value"] = v + time.sleep(0.2) + + sampler_thread = threading.Thread(target=sampler, daemon=True) + sampler_thread.start() + + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(one, range(10))) + + stop.set() + sampler_thread.join(timeout=2) + + check(not errors, "parallel uploads errors: %s" % errors[:3]) + print(" 10 параллельных загрузок, пик RSS=%s MiB" % peak["value"]) + if peak["value"]: + check(peak["value"] < 400, "peak RSS %s MiB слишком высок" % peak["value"]) + + +# ---------- main ---------- + +def main(): + args = arguments("Lab4 level 4 checks") + c = Client(args.base_url, args.ca_file) + + print("=== oversize ===") + test_oversize(c) + print("PASS: oversize -> 413") + + print("=== corrupted image ===") + test_corrupted_image(c) + print("PASS: corrupted -> 415") + + print("=== spoofed MIME ===") + test_mime_spoof(c) + print("PASS: spoofed MIME -> 415") + + print("=== bad filename ===") + test_bad_filename(c) + print("PASS: bad filename -> 400") + + print("=== foreign attachment / cross-chat ===") + test_foreign_attachment_and_cross_chat(c) + print("PASS: foreign -> 404, cross-chat -> 404") + + print("=== attachment belongs to chat ===") + test_attachment_belongs_to_chat(c) + print("PASS: attachment from another chat -> 404") + + print("=== compensation on DB failure ===") + test_compensation_on_db_failure(c) + print("PASS: компенсация orphan-объектов при отказе БД") + + print("=== parallel uploads memory ===") + test_parallel_uploads_memory(c) + print("PASS: 10 параллельных загрузок в пределах лимита памяти") + + print("PASS: all lab4 level-4 checks passed") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab5_test_level4.py b/student-tests/lab5_test_level4.py new file mode 100644 index 0000000..8bed82b --- /dev/null +++ b/student-tests/lab5_test_level4.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python3 +"""Level 4 для lab5: варианты, outbox, идемпотентность, DLQ, lease, retry. + +Запуск: + $env:PYTHONUTF8="1" + python student-tests/lab5_test_level4.py --base-url http://localhost:8080 + +Требует рабочий стенд из compose.yaml (LB + 2 api + worker + outbox + +RabbitMQ + MinIO + PostgreSQL + Redis). Использует psql/mc через +docker run в compose-сети, как lab4_test_level4.py. + +Сценарии: + === variants === + Все 4 варианта: original, thumbnail (256×160), crop (128×128), + optimized (WebP). Реальные bytes и размеры, а не JSON-обещания. + === outbox crash between COMMIT and publish === + Остановить outbox → upload → строка outbox есть, published_at NULL + → запустить outbox → published_at заполнено, attachment готов. + === duplicate delivery does not create duplicates === + Ручная повторная публикация того же job → variants не дублируются. + === worker stopped: 202 → content 409 → ready === + Остановить worker → upload → 202 queued → content 409 → старт + worker → ready и thumbnail доступен. + === broken staging → retry_exhausted → failed + DLQ === + Удалить staging-объект → worker исчерпает retry → failed + error_code=retry_exhausted + сообщение в DLQ. + === stuck processing (lease) → queued → ready === + Симулировать processing с истёкшим lease → recover (через s3_cleanup + или через явный SQL) возвращает в queued → worker обрабатывает → ready. +""" +import hashlib +import io +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = ROOT / "tests" / "fixtures" / "sample.png" +COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab5_default") +PG_CONTAINER = os.environ.get("PG_CONTAINER", "mtuci-postgres") +RABBIT_CONTAINER = os.environ.get("RABBIT_CONTAINER", "mtuci-rabbitmq") +MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920") + + +# ---------- helpers ---------- + +def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str): + qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote) + url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs + req = urllib.request.Request( + url, data=data, method="POST", + headers={ + "Authorization": "Bearer " + actor["token"], + "Content-Type": ctype, + }, + ) + try: + with c.opener.open(req, timeout=30) as resp: + return resp.code, json.loads(resp.read()), resp.headers + except urllib.error.HTTPError as e: + body = e.read() + try: + return e.code, json.loads(body), e.headers + except Exception: + return e.code, None, e.headers + + +def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"): + return _raw_upload(c, actor, chat["id"], filename, data, ctype) + + +def _env_from_file(): + env = os.environ.copy() + env_file = ROOT / ".env" + if env_file.is_file(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip()) + return env + + +def _psql(sql: str) -> str: + """psql через docker compose exec в mtuci-postgres.""" + r = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "high_load_messenger", + "-tA", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=str(ROOT), capture_output=True, text=True, timeout=60, + ) + check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr) + return r.stdout.strip() + + +def _compose(action: str, service: str): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", action, service], + cwd=str(ROOT), capture_output=True, text=True, timeout=90, + ) + + +def _wait_ready(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + return + except Failure: + time.sleep(1) + raise Failure("API не вернулся в ready за %d секунд" % timeout) + + +def _mc(*args: str) -> subprocess.CompletedProcess: + """Запуск mc через одноразовый контейнер в compose-сети.""" + env = _env_from_file() + user = env.get("MINIO_ROOT_USER", "minioadmin") + password = env.get("MINIO_ROOT_PASSWORD", "minioadmin") + mc_host = "http://%s:%s@minio:9000" % (user, password) + + ins = subprocess.run( + ["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"], + capture_output=True, text=True, timeout=10, + ) + ep = (ins.stdout or "").strip() + prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"] + + return subprocess.run( + ["docker", "run", "--rm", "--network", COMPOSE_NETWORK, + "-e", "MC_HOST_local=" + mc_host, MC_IMAGE] + + prefix + list(args), + capture_output=True, text=True, timeout=60, + ) + + +def _mc_rm(key: str) -> None: + env = _env_from_file() + bucket = env.get("S3_BUCKET", "messenger-attachments") + r = _mc("rm", "local/%s/%s" % (bucket, key)) + check(r.returncode == 0, "mc rm failed: %s" % r.stderr[:300]) + + +def _dlq_count() -> int: + r = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", + "rabbitmqctl", "list_queues", "name", "messages"], + cwd=str(ROOT), capture_output=True, text=True, timeout=30, + ) + check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300]) + for line in r.stdout.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] == "attachment.process.v1.dlq": + return int(parts[1]) + return 0 + + +def _queue_consumers(qname: str) -> int: + r = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", + "rabbitmqctl", "list_queues", "name", "consumers"], + cwd=str(ROOT), capture_output=True, text=True, timeout=30, + ) + check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300]) + for line in r.stdout.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] == qname: + return int(parts[1]) + return 0 + + +# ---------- 1. Все 4 варианта ---------- + +def test_variants(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + status, att, _h = _upload(c, alice, chat, data) + check(status == 202, "upload must be 202, got %s" % status) + check(att["status"] == "queued", "upload status must be queued, got %s" % att["status"]) + + ready = c.wait_ready(alice, att, wait_seconds=60) + check(ready["status"] == "ready", "must be ready") + check(ready["error_code"] is None, "ready must have error_code=null") + + names = sorted(v["name"] for v in ready["variants"]) + check(names == ["crop", "optimized", "thumbnail"], + "expected variants thumbnail+crop+optimized, got %s" % names) + + # original + download = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob) + check(download["bytes"] == data, "original bytes differ") + check(download["headers"].get_content_type() == "image/png", "original content-type") + + # thumbnail: JPEG 256x160, sha256, size + v = next(x for x in ready["variants"] if x["name"] == "thumbnail") + thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob) + check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be image/jpeg") + check(len(thumb["bytes"]) == v["size_bytes"], "thumbnail size_bytes mismatch") + check(hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], "thumbnail sha256 mismatch") + w, h = jpeg_size(thumb["bytes"]) + check((w, h) == (256, 160), "thumbnail dimensions %sx%s, expected 256x160" % (w, h)) + check((v["width"], v["height"]) == (256, 160), "thumbnail metadata dimensions mismatch") + + # crop: JPEG 128x128 + v = next(x for x in ready["variants"] if x["name"] == "crop") + crop = c.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=bob) + check(crop["headers"].get_content_type() == "image/jpeg", "crop must be image/jpeg") + check(len(crop["bytes"]) == v["size_bytes"], "crop size_bytes mismatch") + w, h = jpeg_size(crop["bytes"]) + check((w, h) == (128, 128), "crop dimensions %sx%s, expected 128x128" % (w, h)) + + # optimized: WebP + v = next(x for x in ready["variants"] if x["name"] == "optimized") + opt = c.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=bob) + check(opt["headers"].get_content_type() == "image/webp", "optimized must be image/webp") + check(len(opt["bytes"]) == v["size_bytes"], "optimized size_bytes mismatch") + check(hashlib.sha256(opt["bytes"]).hexdigest() == v["sha256"], "optimized sha256 mismatch") + # WebP RIFF-заголовок + check(opt["bytes"][:4] == b"RIFF" and opt["bytes"][8:12] == b"WEBP", "optimized is not WebP") + + +# ---------- 2. Outbox: crash между COMMIT и publish ---------- + +def test_outbox_crash(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + + # Останавливаем outbox, чтобы job застряла в БД после COMMIT. + r = _compose("stop", "outbox") + check(r.returncode == 0, "stop outbox failed: %s" % r.stderr[:300]) + + try: + status, att, _h = _upload(c, alice, chat, data) + check(status == 202, "upload must be 202 with stopped outbox, got %s" % status) + + # В БД должна быть строка outbox с published_at IS NULL. + unpublished = int(_psql( + "SELECT count(*) FROM outbox WHERE published_at IS NULL" + " AND payload->>'attachment_id' = '%s';" % att["id"] + )) + check(unpublished == 1, + "expected 1 unpublished outbox row for %s, got %d" % (att["id"], unpublished)) + + # attachment остаётся queued. + st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"]) + check(st == "queued", "attachment must stay queued while outbox stopped, got %s" % st) + finally: + r = _compose("start", "outbox") + check(r.returncode == 0, "start outbox failed: %s" % r.stderr[:300]) + + # После старта outbox job должна быть опубликована и обработана. + ready = c.wait_ready(alice, att, wait_seconds=90) + check(ready["status"] == "ready", "outbox recovery: attachment must become ready") + published = int(_psql( + "SELECT count(*) FROM outbox WHERE published_at IS NOT NULL" + " AND payload->>'attachment_id' = '%s';" % att["id"] + )) + check(published == 1, "outbox row must be marked published, got %d" % published) + + +# ---------- 3. Повторная доставка не создаёт дублей ---------- + +def test_duplicate_delivery_no_duplicates(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + status, att, _h = _upload(c, alice, chat, data) + check(status == 202, "upload must be 202, got %s" % status) + ready = c.wait_ready(alice, att, wait_seconds=60) + check(ready["status"] == "ready", "attachment must become ready") + + # Ровно 3 non-original варианта в БД. + n_before = int(_psql( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"] + )) + check(n_before == 3, "expected exactly 3 variants, got %d" % n_before) + + # Повторно публикуем тот же job через ручную публикацию в RabbitMQ + # с тем же payload (эмуляция redelivery после side effect до ack). + payload_json = _psql( + "SELECT payload::text FROM outbox WHERE payload->>'attachment_id' = '%s' LIMIT 1;" % att["id"] + ) + check(bool(payload_json), "payload not found in outbox") + # Публикуем в основную очередь через pika в контейнере worker. + script = ( + "import os, json, pika\n" + "url = os.environ['RABBITMQ_URL']\n" + "params = pika.URLParameters(url)\n" + "conn = pika.BlockingConnection(params)\n" + "ch = conn.channel()\n" + "ch.confirm_delivery()\n" + "ch.exchange_declare('messenger.attachments', exchange_type='direct', durable=True)\n" + "body = %r\n" + "ch.basic_publish(exchange='messenger.attachments', routing_key='attachment.process.v1',\n" + " body=body.encode('utf-8'),\n" + " properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'),\n" + " mandatory=True)\n" + "conn.close()\n" + "print('PUBLISHED')\n" % payload_json + ) + r = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "worker", + "python", "-c", script], + cwd=str(ROOT), capture_output=True, text=True, timeout=30, + ) + check(r.returncode == 0 and "PUBLISHED" in r.stdout, + "manual duplicate publish failed: %s" % (r.stderr[:300] or r.stdout[:300])) + + # Даём worker-у обработать дубль. + time.sleep(5) + + n_after = int(_psql( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"] + )) + check(n_after == n_before, + "duplicate delivery created extra variants: %d -> %d" % (n_before, n_after)) + # Всё ещё ready и не failed. + st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"]) + check(st == "ready", "duplicate must not change status, got %s" % st) + + +# ---------- 4. Остановка worker: 202 → 409 → ready ---------- + +def test_worker_stopped(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + + r = _compose("stop", "worker") + check(r.returncode == 0, "stop worker failed: %s" % r.stderr[:300]) + try: + # Upload должен вернуть 202, attachment = queued, content — 409. + status, att, _h = _upload(c, alice, chat, data) + check(status == 202, "upload with stopped worker must be 202, got %s" % status) + check(att["status"] == "queued", "attachment must be queued") + + # content должен вернуть 409, пока не ready. + r_content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], + (409, 404), actor=bob) + check(r_content["status"] == 409, + "content must be 409 for queued, got %s" % r_content["status"]) + + # API остаётся доступен для других эндпоинтов. + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + finally: + r = _compose("start", "worker") + check(r.returncode == 0, "start worker failed: %s" % r.stderr[:300]) + + # Ждём consumer_started и processing. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if _queue_consumers("attachment.process.v1") >= 1: + break + time.sleep(1) + else: + raise Failure("worker не подключился к очереди за 30 секунд") + + ready = c.wait_ready(alice, att, wait_seconds=90) + check(ready["status"] == "ready", "after worker start attachment must become ready") + + # thumbnail доступен + thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob) + check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail content-type") + + +# ---------- 5. Broken staging → retry_exhausted → failed + DLQ ---------- + +def test_broken_staging_dlq(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + + # Останавливаем worker, чтобы сначала зафиксировать staging_key в БД, + # затем удалить staging-объект и запустить worker. + r = _compose("stop", "worker") + check(r.returncode == 0, "stop worker failed") + try: + status, att, _h = _upload(c, alice, chat, data) + check(status == 202, "upload must be 202") + + # Удаляем staging-объект из S3, чтобы worker не смог его прочитать. + staging = _psql( + "SELECT object_key FROM attachments WHERE id = '%s';" % att["id"] + ) + check(staging.startswith("staging/"), + "queued attachment object_key must be staging/*, got %r" % staging) + _mc_rm(staging) + finally: + r = _compose("start", "worker") + check(r.returncode == 0, "start worker failed") + + dlq_before = _dlq_count() + + # Ждём финального failed (retry: 5+15+60 ≈ 80c + обработка). + deadline = time.monotonic() + 180 + final_status = None + error_code = None + while time.monotonic() < deadline: + st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"]) + if st == "failed": + final_status = st + error_code = _psql( + "SELECT coalesce(error_code, '') FROM attachments WHERE id = '%s';" % att["id"] + ) + break + if st == "ready": + raise Failure("attachment unexpectedly became ready: staging was deleted") + time.sleep(5) + + check(final_status == "failed", + "expected failed after retry exhaustion, got %s" % final_status) + check(error_code == "retry_exhausted", + "expected error_code=retry_exhausted, got %r" % error_code) + + # DLQ должен содержать хотя бы одно сообщение. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if _dlq_count() > dlq_before: + break + time.sleep(1) + else: + raise Failure("DLQ did not receive the message after retry exhaustion " + "(dlq_before=%d)" % dlq_before) + + +# ---------- 6. Stuck processing (lease) → queued → ready ---------- + +def test_stuck_processing_recovered(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + + # Останавливаем worker, чтобы job осталась queued и мы могли её «сломать». + r = _compose("stop", "worker") + check(r.returncode == 0, "stop worker failed") + try: + status, att, _h = _upload(c, alice, chat, data) + check(status == 202, "upload must be 202, got %s" % status) + + # Симулируем зависший processing: status=processing, lease в прошлом. + _psql( + "UPDATE attachments SET status='processing'," + " processing_until = now() - interval '5 minutes'," + " updated_at = now() WHERE id = '%s';" % att["id"] + ) + st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"]) + check(st == "processing", "setup failed, expected processing, got %s" % st) + finally: + r = _compose("start", "worker") + check(r.returncode == 0, "start worker failed") + + # Проверяем, что SQL-recover (эквивалент recover_stuck_processing) + # переводит запись в queued. Это и есть логика, которую вызывает + # s3_cleanup по интервалу. Здесь дёргаем явно, чтобы не зависеть от + # CLEANUP_INTERVAL_SECONDS. + recovered = int(_psql( + "WITH upd AS (" + " UPDATE attachments SET status='queued', processing_until=NULL, updated_at=now()" + " WHERE status='processing' AND processing_until IS NOT NULL" + " AND processing_until < now()" + " RETURNING id" + ") SELECT count(*) FROM upd;" + )) + check(recovered >= 1, "recover_stuck_processing did not return any row") + + # worker после старта подхватит queued и доведёт до ready. + ready = c.wait_ready(alice, att, wait_seconds=90) + check(ready["status"] == "ready", "recovered attachment must become ready") + + +# ---------- main ---------- + +def main(): + args = arguments("Lab5 level 4 checks") + c = Client(args.base_url, args.ca_file) + c.call("GET", "/health/ready", schema="Health") + + print("=== variants (thumbnail/crop/optimized) ===") + test_variants(c) + print("PASS: original, thumbnail 256×160, crop 128×128, optimized WebP") + + print("=== outbox crash between COMMIT and publish ===") + test_outbox_crash(c) + print("PASS: outbox recovers job after stop/start without manual intervention") + + print("=== duplicate delivery does not create duplicates ===") + test_duplicate_delivery_no_duplicates(c) + print("PASS: re-delivery of the same job is a no-op") + + print("=== worker stopped: 202 → content 409 → ready ===") + test_worker_stopped(c) + print("PASS: accepted work survives worker restart") + + print("=== broken staging → retry_exhausted → failed + DLQ ===") + test_broken_staging_dlq(c) + print("PASS: bounded retry, failed with safe error_code, message in DLQ") + + print("=== stuck processing (lease) → queued → ready ===") + test_stuck_processing_recovered(c) + print("PASS: expired lease is recovered, same attachment becomes ready") + + print("PASS: all lab5 level-4 checks passed") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/lab6_test_level4.py b/student-tests/lab6_test_level4.py new file mode 100644 index 0000000..9177afb --- /dev/null +++ b/student-tests/lab6_test_level4.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +"""Level 4 для lab6: сквозной trace API→broker→worker, метрики, логи, dashboard, +alert rules, поведение при отказе telemetry-backend. + +Запуск: + $env:PYTHONUTF8="1" + python student-tests/lab6_test_level4.py --base-url http://localhost:8080 +""" +import base64 +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = ROOT / "tests" / "fixtures" / "sample.png" + +GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost:3000") +GRAFANA_ADMIN_USER = os.environ.get("GRAFANA_ADMIN_USER", "admin") +GRAFANA_ADMIN_PASSWORD = os.environ.get("GRAFANA_ADMIN_PASSWORD", "admin") + +READY_TIMEOUT = int(os.environ.get("READY_TIMEOUT", "180")) + + +def _compose_exec(service: str, *args: str, timeout: int = 30): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", service, *args], + cwd=str(ROOT), capture_output=True, text=True, timeout=timeout, + ) + + +def _compose(action: str, service: str, timeout: int = 90): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", action, service], + cwd=str(ROOT), capture_output=True, text=True, timeout=timeout, + ) + + +def _compose_logs(service: str, tail: int = 20) -> str: + r = subprocess.run( + ["docker", "compose", "--env-file", ".env", "logs", "--no-color", + "--tail=%d" % tail, service], + cwd=str(ROOT), capture_output=True, text=True, timeout=30, + ) + return (r.stdout or "") + (r.stderr or "") + + +def _http_get(url: str, timeout: float = 10.0, basic_auth: tuple | None = None): + headers = {"Accept": "application/json"} + if basic_auth is not None: + token = base64.b64encode(("%s:%s" % basic_auth).encode()).decode() + headers["Authorization"] = "Basic " + token + req = urllib.request.Request(url, headers=headers, method="GET") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace") + except (urllib.error.URLError, OSError) as e: + return 0, "%s: %s" % (type(e).__name__, e) + + +def _prom_query(query: str) -> dict: + qs = "query=" + urllib.parse.quote(query, safe="") + r = _compose_exec( + "prometheus", "sh", "-c", + "wget -qO- 'http://localhost:9090/api/v1/query?%s'" % qs, + ) + if r.returncode != 0 or not r.stdout.strip(): + raise Failure("prometheus query failed: %s" % (r.stderr[:300] or r.stdout[:300])) + try: + return json.loads(r.stdout) + except Exception: + raise Failure("prometheus non-JSON: %s" % r.stdout[:300]) + + +def _tempo_search(query: str, since_seconds: int = 300, limit: int = 20) -> list[dict]: + now = int(time.time()) + start = now - since_seconds + qs = "q=%s&start=%d&end=%d&limit=%d" % ( + urllib.parse.quote(query, safe=""), start, now, limit, + ) + r = _compose_exec( + "tempo", "sh", "-c", + "wget -qO- 'http://localhost:3200/api/search?%s'" % qs, + ) + if r.returncode != 0 or not r.stdout.strip(): + return [] + try: + data = json.loads(r.stdout) + except Exception: + return [] + return data.get("traces", []) or [] + + +def _tempo_get_trace(trace_id: str) -> dict: + r = _compose_exec( + "tempo", "sh", "-c", + "wget -qO- 'http://localhost:3200/api/traces/%s'" % trace_id, + ) + if r.returncode != 0 or not r.stdout.strip(): + raise Failure("tempo trace fetch failed: %s" % (r.stderr[:300] or r.stdout[:300])) + return json.loads(r.stdout) + + +def _tempo_span_names(trace: dict) -> set[str]: + names: set[str] = set() + for batch in trace.get("batches", []): + scopes = batch.get("scopeSpans") or batch.get("instrumentationLibrarySpans") or [] + for scope in scopes: + for span in scope.get("spans", []) or []: + names.add(span.get("name", "")) + return names + + +def _loki_query(query: str, since_seconds: int = 300, limit: int = 200) -> dict: + end = int(time.time() * 1e9) + start = end - since_seconds * 1_000_000_000 + qs = "query=%s&start=%d&end=%d&limit=%d" % ( + urllib.parse.quote(query, safe=""), start, end, limit, + ) + r = _compose_exec( + "loki", "sh", "-c", + "wget -qO- 'http://localhost:3100/loki/api/v1/query_range?%s'" % qs, + ) + if r.returncode != 0 or not r.stdout.strip(): + raise Failure("loki query failed: %s" % (r.stderr[:300] or r.stdout[:300])) + try: + return json.loads(r.stdout) + except Exception: + raise Failure("loki returned non-JSON: %s" % r.stdout[:300]) + + +def _raw_upload(c: Client, actor, chat_id: str, filename: str, + data: bytes, ctype: str): + qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote) + url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs + req = urllib.request.Request( + url, data=data, method="POST", + headers={ + "Authorization": "Bearer " + actor["token"], + "Content-Type": ctype, + }, + ) + try: + with c.opener.open(req, timeout=30) as resp: + return resp.code, json.loads(resp.read()), resp.headers + except urllib.error.HTTPError as e: + body = e.read() + try: + return e.code, json.loads(body), e.headers + except Exception: + return e.code, None, e.headers + + +def _wait_ready(c, actor, att, timeout: int) -> dict: + """Обёртка с диагностикой: если долго не ready — показываем статус и логи.""" + deadline = time.monotonic() + timeout + last_status = None + while time.monotonic() < deadline: + a = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, + schema="Attachment")["json"] + last_status = a.get("status") + if last_status == "ready": + return a + if last_status == "failed": + raise Failure("attachment failed: error_code=%r" % a.get("error_code")) + time.sleep(0.5) + diag = ( + "\nlast_status=%r (waited %ss)\n--- outbox logs ---\n%s\n--- worker logs ---\n%s" + % (last_status, timeout, _compose_logs("outbox", 20), + _compose_logs("worker", 20)) + ) + raise Failure("attachment did not become ready" + diag) + + +def _wait_trace_with_spans(since_seconds: int, need_substrings: list[str], + timeout: float = 180.0) -> dict: + deadline = time.monotonic() + timeout + last_seen: list[dict] = [] + while time.monotonic() < deadline: + for svc in ("messenger-api", "messenger-worker", "messenger-outbox"): + candidates = _tempo_search( + '{resource.service.name = "%s"}' % svc, + since_seconds=since_seconds, + ) + for t in candidates: + trace_id = t.get("traceID") + if not trace_id: + continue + try: + full = _tempo_get_trace(trace_id) + except Failure: + continue + names = _tempo_span_names(full) + last_seen.append({"trace_id": trace_id, "names": sorted(names)}) + if all(any(s in n for n in names) for s in need_substrings): + return {"trace_id": trace_id, "names": sorted(names)} + time.sleep(3) + tail = "; ".join( + "%s -> %s" % (x["trace_id"][:8], x["names"][:6]) for x in last_seen[:5] + ) + raise Failure( + "не дождались trace со span-names, содержащими %s за %ss; последние trace: %s" + % (need_substrings, timeout, tail or "нет") + ) + + +def test_trace_api_to_worker(c): + alice, bob, eve, chat = c.bootstrap() + data = SAMPLE.read_bytes() + started = int(time.time()) + + status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png") + check(status == 202, "upload must be 202, got %s" % status) + check(att["status"] == "queued", "upload status must be queued") + + ready = _wait_ready(c, alice, att, timeout=READY_TIMEOUT) + check(ready["status"] == "ready", "attachment must become ready") + + since = max(60, int(time.time()) - started + 180) + trace = _wait_trace_with_spans( + since_seconds=since, + need_substrings=[ + "/api/v1/chats/{chat_id}/attachments", + "job.publish", + "job.consume", + ], + timeout=180, + ) + print(" trace_id:", trace["trace_id"]) + print(" spans:", ", ".join(trace["names"])) + + +def test_metrics_in_prometheus(c): + alice, bob, eve, chat = c.bootstrap() + c.message(alice, chat, "metrics-" + uuid.uuid4().hex) + c.message(alice, chat, "metrics-" + uuid.uuid4().hex) + data = SAMPLE.read_bytes() + status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png") + check(status == 202, "upload must be 202") + _wait_ready(c, alice, att, timeout=READY_TIMEOUT) + + deadline = time.monotonic() + 120 + required = { + "http_requests_total", + "http_request_duration_seconds_bucket", + "jobs_processed_total", + "job_processing_duration_seconds_bucket", + } + got: set[str] = set() + while time.monotonic() < deadline: + got = set() + for name in required: + r = _prom_query("count(%s)" % name) + series = r.get("data", {}).get("result", []) + if series: + got.add(name) + if got == required: + break + time.sleep(3) + missing = required - got + check(not missing, "Prometheus missing metrics: %s" % sorted(missing)) + print(" metrics in Prometheus:", sorted(got)) + + r = _prom_query("count by (http_route, http_status_class) (http_requests_total)") + series = r.get("data", {}).get("result", []) + check(series, "http_requests_total has no series") + import re as _re + for s in series: + metric = s.get("metric", {}) + route = metric.get("http_route", "") + check(route.startswith("/api/") or route.startswith("/health"), + "unexpected http_route: %r" % route) + check(_re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + route) is None, + "http_route contains raw UUID: %r" % route) + + r = _prom_query("sum(jobs_processed_total{outcome=\"success\"})") + series = r.get("data", {}).get("result", []) + check(series, "jobs_processed_total{outcome=success} absent") + value = float(series[0]["value"][1]) + check(value >= 1, "jobs success metric is %r" % value) + print(" jobs_processed_total{outcome=success} =", value) + + +def test_logs_in_loki(c): + time.sleep(15) + + deadline = time.monotonic() + 90 + streams = [] + while time.monotonic() < deadline: + r = _loki_query('{service_name="messenger-api"} |= "event=http"', + since_seconds=300) + streams = r.get("data", {}).get("result", []) + if streams: + break + time.sleep(3) + check(streams, "Loki: no messenger-api logs with event=http") + lines = [v[1] for s in streams for v in s.get("values", [])] + check(any("request_id=" in line for line in lines), + "API logs missing request_id") + + r = _loki_query('{service_name="messenger-worker"} |= "event="', since_seconds=300) + check(r.get("data", {}).get("result", []), + "Loki: no messenger-worker logs") + + # Ищем запись event=http с ненулевым trace_id (корреляция лог↔trace). + deadline = time.monotonic() + 90 + ok = False + while time.monotonic() < deadline: + r = _loki_query( + '{service_name=~"messenger-api|messenger-worker"} ' + '|~ "event=(http|job_ready)" |~ "trace_id=" ', + since_seconds=300, + ) + streams = r.get("data", {}).get("result", []) + for s in streams: + for v in s.get("values", []): + line = v[1] + idx = line.find("trace_id=") + if idx < 0: + continue + tail = line[idx + len("trace_id="):] + tid = tail.split()[0].strip() + if len(tid) == 32 and set(tid) != {"0"}: + ok = True + break + if ok: + break + if ok: + break + time.sleep(3) + check(ok, "Loki: no logs with non-zero trace_id (log↔trace correlation)") + print(" Loki OK: api+worker logs with non-zero trace_id found") + + +def _grafana_ds_uids(auth) -> dict[str, dict]: + code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth) + check(code == 200, "grafana datasources API %s" % code) + return {d.get("uid"): d for d in json.loads(body)} + + +def test_grafana_provisioning(c): + deadline = time.monotonic() + 60 + auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD) + while time.monotonic() < deadline: + code, _ = _http_get(GRAFANA_URL + "/api/health") + if code == 200: + break + time.sleep(2) + else: + raise Failure("Grafana API не поднялась за 60 с") + + by_uid = _grafana_ds_uids(auth) + # Требуемые UID явно заданы в provisioning и совпадают со ссылками в + # dashboard/rules/derived fields. + for uid, type_ in (("prometheus", "prometheus"), + ("loki", "loki"), + ("tempo", "tempo")): + check(uid in by_uid, "Grafana missing datasource uid=%r (have %s)" + % (uid, sorted(by_uid))) + check(by_uid[uid].get("type") == type_, + "datasource uid=%r has wrong type: %r" + % (uid, by_uid[uid].get("type"))) + print(" datasources by uid:", sorted(by_uid)) + + code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6", + basic_auth=auth) + check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code) + dash = json.loads(body) + dashboard = dash.get("dashboard", {}) + panels = dashboard.get("panels", []) + titles = {p.get("title") for p in panels} + check(any("p95" in (t or "") for t in titles), + "dashboard missing p95 panel") + check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles), + "dashboard missing jobs panel") + + # Проверяем, что панель логов ссылается на существующий uid Loki, + # а не на «магическое» имя. + for p in panels: + ds = p.get("datasource") or {} + uid = ds.get("uid") + if uid is not None: + check(uid in by_uid, + "panel %r references unknown datasource uid %r" + % (p.get("title"), uid)) + print(" dashboard panels:", sorted(t for t in titles if t)) + + code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules", + basic_auth=auth) + check(code == 200, "grafana alert-rules API %s" % code) + rules = json.loads(body) + check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules)) + + for r in rules: + # datasourceUid в data должен существовать. + for item in r.get("data", []): + uid = item.get("datasourceUid") + if uid and uid != "__expr__": + check(uid in by_uid, + "rule %r references unknown datasource uid %r" + % (r.get("title"), uid)) + + # Все правила должны быть health=ok. Если datasources не согласованы, + # Grafana вернёт health=error "data source not found", как было + # в замечании проверяющего. + deadline = time.monotonic() + 180 + while True: + code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules", + basic_auth=auth) + check(code == 200, "grafana alert-rules API %s" % code) + rules = json.loads(body) + bad = [r.get("title") for r in rules + if r.get("health") not in (None, "ok")] + if not bad: + break + if time.monotonic() > deadline: + raise Failure("Grafana alert rules with non-ok health: %s" % bad) + time.sleep(3) + print(" alert rules:", [r.get("title") for r in rules]) + + +def _prom_alerts() -> list[dict]: + r = _compose_exec("prometheus", "sh", "-c", + "wget -qO- 'http://localhost:9090/api/v1/alerts'") + if r.returncode != 0 or not r.stdout.strip(): + return [] + try: + data = json.loads(r.stdout) + except Exception: + return [] + return data.get("data", {}).get("alerts", []) or [] + + +def _wait_alert(name: str, state: str, timeout: float = 240.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + alerts = _prom_alerts() + # Prometheus /api/v1/alerts возвращает только pending/firing. + # Состояние "inactive" == алерта с таким alertname нет в списке. + active = [ + a for a in alerts + if a.get("labels", {}).get("alertname") == name + ] + if state == "inactive": + if not active: + return + else: + for a in active: + if a.get("state") == state: + return + time.sleep(3) + raise Failure("alert %s never reached %s" % (name, state)) + + +def test_alert_firing_and_resolved(c): + alice, bob, eve, chat = c.bootstrap() + + stopped = _compose("stop", "postgres") + check(stopped.returncode == 0, "stop postgres failed") + + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call("GET", "/api/v1/users/me", (401, 503), actor=alice) + except Failure: + pass + time.sleep(0.3) + _wait_alert("MessengerHighErrorRate", "firing", timeout=240) + print(" MessengerHighErrorRate firing") + finally: + started = _compose("start", "postgres") + check(started.returncode == 0, "start postgres failed") + + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + break + except Failure: + time.sleep(1) + else: + raise Failure("API didn't become ready after postgres restart") + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User") + except Failure: + pass + time.sleep(0.3) + _wait_alert("MessengerHighErrorRate", "inactive", timeout=300) + print(" MessengerHighErrorRate resolved") + + +def test_telemetry_backend_down(c): + for svc in ("tempo", "loki", "prometheus"): + _compose("start", svc) + time.sleep(3) + + stopped = {} + for svc in ("prometheus", "loki", "tempo"): + r = _compose("stop", svc) + check(r.returncode == 0, "stop %s failed: %s" % (svc, r.stderr[:300])) + stopped[svc] = True + + try: + alice, bob, eve, chat = c.bootstrap() + deadline = time.monotonic() + 30 + ok = 0 + while time.monotonic() < deadline: + try: + c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User") + c.message(alice, chat, "telem-down-" + uuid.uuid4().hex[:8]) + ok += 1 + except Failure: + pass + time.sleep(0.2) + check(ok >= 5, "API unavailable while telemetry backend is down (ok=%d)" % ok) + + # В образе otel/opentelemetry-collector-contrib нет coreutils `true`, + # поэтому проверяем живость штатным бинарником самого коллектора + # (тот же приём, что в healthcheck compose.yaml). + ps = _compose_exec("otel-collector", "/otelcol-contrib", "--version") + check(ps.returncode == 0, + "otel-collector died while backends were unavailable") + + stats = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", + "mtuci-otel-collector"], + capture_output=True, text=True, timeout=15, + ) + if stats.returncode == 0: + used = stats.stdout.strip().split("/")[0].strip() + mb = None + if used.endswith("MiB"): + mb = float(used[:-3]) + elif used.endswith("GiB"): + mb = float(used[:-3]) * 1024 + if mb is not None: + check(mb < 260, "otel-collector RSS=%s MiB exceeds limit" % mb) + print(" otel-collector RSS while backends down:", mb, "MiB") + finally: + for svc in stopped: + _compose("start", svc) + time.sleep(3) + + print(" API and Collector remained healthy while backends were down") + + +def main(): + args = arguments("Lab6 level 4 checks") + c = Client(args.base_url, args.ca_file) + c.call("GET", "/health/ready", schema="Health") + + print("=== trace API -> broker -> worker (Tempo) ===") + test_trace_api_to_worker(c) + print("PASS: single trace with upload + job.publish + job.consume") + + print("=== metrics in Prometheus ===") + test_metrics_in_prometheus(c) + print("PASS: http_requests_total + duration + jobs_processed_total present, " + "low-cardinality route labels") + + print("=== logs in Loki ===") + test_logs_in_loki(c) + print("PASS: structured logs of api+worker with trace_id") + + print("=== Grafana provisioning ===") + test_grafana_provisioning(c) + print("PASS: dashboard messenger-lab6 + >=2 alert rules provisioned") + + print("=== alert firing and resolved ===") + test_alert_firing_and_resolved(c) + print("PASS: alert fires on artificial outage and resolves on recovery") + + print("=== telemetry backend down ===") + test_telemetry_backend_down(c) + print("PASS: API works, Collector stays within memory limit") + + print("PASS: all lab6 level-4 checks passed") + + +if __name__ == "__main__": + run(main) \ No newline at end of file