diff --git a/.env.example b/.env.example index 2b32655..3e66113 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,11 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +POSTGRES_PASSWORD=lab6-local-postgres-change-me +S3_ROOT_USER=lab6root +S3_ROOT_PASSWORD=lab6-local-s3-admin-change-me +S3_ACCESS_KEY_ID=lab6api +S3_SECRET_ACCESS_KEY=lab6-local-s3-api-change-me S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +SESSION_TTL_SECONDS=3600 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_ADMIN_PASSWORD=lab6-local-grafana-change-me diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2954f81 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Pinned base image. Use a digest for reproducibility if you have one available: +# FROM python:3.12.7-slim-bookworm@sha256: +FROM python:3.12.7-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=2s --timeout=3s --start-period=5s --retries=2 \ + CMD python -c "import sys,urllib.request; \ +sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)" + +CMD ["python", "-m", "src"] diff --git a/REPORT.md b/REPORT.md index b97ae42..bfaf66c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,80 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Вотчицева Вероника, БВТ2302 +- **Целевая оценка:** 4 (трек на 5 не заявлялся). +- **Commit/PR:** +- **Использованные источники и ИИ:** + - Python 3.12, FastAPI, SQLAlchemy, RabbitMQ (aio-pika), SeaweedFS (boto3). + - OpenTelemetry SDK (api, sdk, exporter-otlp-proto-http). + - Стек мониторинга: OTel Collector, Prometheus, Loki, Tempo, Grafana. + - При подготовке конфигураций и кода использовался ассистент (Claude). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Prerequisites:** Docker Desktop с поддержкой `--wait`, Python 3.9+ для скриптов проверки. + +```powershell +# 1. Подготовка окружения +Copy-Item .env.example .env +# (Заполнить пароли POSTGRES_PASSWORD и GRAFANA_ADMIN_PASSWORD) + +# 2. Запуск всего стека (включая Grafana, Prometheus, Loki, Tempo) +docker compose up -d --build --wait --wait-timeout 180 + +# 3. Проверка API +$env:PYTHONUTF8="1" +python tests/smoke.py + +# 4. Запуск нагрузки (для появления графиков) +python tests/load.py --duration 60 --concurrency 4 --output evidence/load-lab6.json +``` + +**Остановка:** `docker compose down`. **Удаление данных (включая дашборды и базы):** `docker compose down -v`. + +Интерфейсы: +- API (Swagger): `http://localhost:8080/docs` +- Grafana: `http://localhost:3000` (admin / пароль из `.env`) +- Prometheus: `http://localhost:9090` +- Jaeger/Tempo UI: `http://localhost:16686` (если настроен порт) ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +В систему добавлен полный стек наблюдаемости (Observability) на базе OpenTelemetry. Схема сбора данных: `API/Worker → OTLP → OTel Collector → (Prometheus, Loki, Tempo) → Grafana`. + +**Три сигнала (Three Pillars):** +1. **Метрики (Prometheus):** Собираются через OTel Collector. Настроены дашборды для RPS, p50/p95/p99, доли ошибок, глубины очереди (Queue Depth) и возраста старейшей задачи. +2. **Логи (Loki):** Отправляются напрямую через OTLP HTTP. Логи структурированные (JSON), содержат `trace_id` и `span_id` для связки с трассами. +3. **Трассы (Tempo):** Позволяют отследить путь запроса. Реализована передача контекста W3C Trace Context (`traceparent`) через RabbitMQ. API создает задачу, сохраняет контекст в Outbox, Worker извлекает его и продолжает трассу. + +**Защита от перегрузки:** +В конфигурации OTel Collector настроен `memory_limiter` (лимит 96 MiB). В SDK настроены ограниченные очереди и таймауты. Если Tempo или Loki упадут, это не заблокирует основной поток обработки API. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py` | PASS | — | +| Импортируемый Dashboard | Grafana → MTUCI Lab 6 | Все панели отрисовываются | `dashboard.json` в Git | +| Сбор метрик | `tests/load.py` | RPS и Latency видны в Prometheus | Скриншот Grafana | +| Связь Trace → Log | Tempo → клик по Trace ID | Переход в Loki с тем же trace_id | Скриншот Tempo/Loki | +| Сквозная трассировка | Tempo: `messenger-api` → `messenger-worker` | Единый трейс от HTTP до записи в S3 | — | +| Алерт на ошибки 5xx | Остановка БД под нагрузкой | RPS падает, HAProxy отдает 503 | Скриншот Prometheus | +| Алерт на Queue Lag | `incident_worker.py --hold 80` | Загорается `MessengerQueueLagGrowing` | Скриншот Prometheus | +| Отказ телеметрии | `stop loki tempo` + `smoke.py` | API продолжает работать (PASS) | — | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Инцидент 1: Остановка воркера (Queue Lag).** +Скрипт `scripts/incident_worker.py --hold 80` остановил контейнер `worker` и загрузил тестовый PNG. API вернул `202 Accepted` (статус `queued`), файл застрял в очереди. Через 35 секунд в Prometheus сработал алерт `MessengerQueueLagGrowing` (возраст старейшей задачи превысил 20 секунд, удерживался >15с). В Grafana на графике «Очередь» наблюдался рост. После автоматического запуска воркера задача обработалась, статус сменился на `ready`, алерт перешел в `resolved` (Inactive). -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Инцидент 2: Отказ PostgreSQL.** +Во время нагрузочного теста я остановила базу данных (`docker compose stop db`). Эндпоинт `/health/ready` начал возвращать `503`, из-за чего **HAProxy исключил реплики API из балансировки** и начал отдавать свой `503` на уровне LB. Из-за этого ошибки 5xx не попали в метрику API (ограничение архитектуры SLI), однако в Grafana график RPS упал до нуля. После команды `docker compose start db` сервис автоматически восстановился. -## Ограничения и следующие шаги +**Инцидент 3: Отказ Telemetry Backend.** +При остановке контейнеров `loki` и `tempo` бизнес-логика приложения не пострадала. Публичный тест `smoke.py` успешно прошел (PASS). В Grafana перестали появляться новые логи и трассы, так как Collector не мог их экспортировать. Это доказывает, что падение системы мониторинга не ломает основной сервис (graceful degradation телеметрии). -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения + +- **Single-node:** Prometheus, Loki, Tempo и Grafana запущены в одном экземпляре. При падении всего Docker-хоста данные мониторинга будут потеряны (хотя volumes сохранят историю при перезапуске). +- **Ограничение метрик HAProxy:** Так как HAProxy отсекает трафик при падении API, ошибки 5xx не всегда попадают в метрики FastAPI. Для полной картины нужен экспортер метрик самого HAProxy (например, `haproxy_exporter`). +- **Кардинальность:** В метках (labels) нет пользовательских UUID, поэтому кардинальность метрик низкая. Все детали запросов (кто именно делал) ищутся через логи и трассы. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..6d9c724 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,234 @@ +x-api: &api + build: . + image: mtusi-lab5-votchitseva-api:local + environment: &api_env + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: postgresql+asyncpg://user:${POSTGRES_PASSWORD:?Copy .env.example to .env}@db:5432/messenger + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: amqp://guest:guest@broker:5672/ + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + s3: + condition: service_healthy + broker: + condition: service_healthy + networks: [front, data] + cpus: 0.5 + mem_limit: 384m + stop_grace_period: 15s + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4).close()"] + interval: 5s + timeout: 5s + retries: 16 + start_period: 35s + +services: + api1: + <<: *api + api2: + <<: *api + + worker: + <<: *api + command: ["python", "-m", "src.worker"] + environment: + <<: *api_env + OTEL_SERVICE_NAME: messenger-worker + healthcheck: + disable: true + deploy: + replicas: 1 + + broker: + image: rabbitmq:3.13-alpine + hostname: rabbitmq + entrypoint: ["/bin/sh", "-c", "rm -f /var/lib/rabbitmq/.erlang.cookie && exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server"] + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + volumes: + - broker_data:/var/lib/rabbitmq + networks: [data] + cpus: 0.5 + mem_limit: 512m + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + + s3_config: + image: python:3.12.7-slim-bookworm + command: ["python", "/bootstrap/generate-s3-config.py"] + environment: + S3_ROOT_USER: ${S3_ROOT_USER:?Set S3_ROOT_USER in .env} + S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Set S3_ROOT_PASSWORD in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - ./infra/generate-s3-config.py:/bootstrap/generate-s3-config.py:ro + - s3_config_data:/s3-config + networks: [data] + restart: "no" + + s3: + image: chrislusf/seaweedfs:4.35 + command: ["mini", "-dir=/data", "-s3.config=/s3-config/s3.json", "-bucket=messenger-attachments"] + volumes: + - s3_data:/data + - s3_config_data:/s3-config:ro + depends_on: + s3_config: {condition: service_completed_successfully} + networks: [data] + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8333/status || exit 1"] + interval: 2s + timeout: 3s + retries: 15 + start_period: 10s + + migrate: + build: . + image: mtusi-lab5-votchitseva-api:local + command: ["python", "-m", "src.migrate"] + environment: + DATABASE_URL: postgresql+asyncpg://user:${POSTGRES_PASSWORD:?Copy .env.example to .env}@db:5432/messenger + depends_on: + db: + condition: service_healthy + networks: [data] + cpus: 0.5 + mem_limit: 256m + healthcheck: + disable: true + + lb: + image: haproxy:3.0.5-alpine + ports: ["127.0.0.1:8080:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + networks: [front] + cpus: 0.25 + mem_limit: 128m + + db: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: user + POSTGRES_DB: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Copy .env.example to .env} + volumes: [pgdata:/var/lib/postgresql/data] + networks: [data] + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: ["CMD-SHELL", "pg_isready -U user -d messenger"] + interval: 2s + timeout: 2s + retries: 15 + + redis: + image: redis:7.4.1-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: [redisdata:/data] + networks: [data] + cpus: 0.25 + mem_limit: 128m + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 15 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.138.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + networks: [data] + restart: unless-stopped + mem_limit: 192m + + prometheus: + image: prom/prometheus:v3.6.0 + command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.enable-lifecycle"] + volumes: + - ./infra/otel/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + ports: ["127.0.0.1:9090:9090"] + networks: [data, front] + restart: unless-stopped + + loki: + image: grafana/loki:3.5.5 + user: "0" + command: ["-config.file=/etc/loki/config.yaml"] + volumes: + - ./infra/otel/loki/config.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + networks: [data] + restart: unless-stopped + + tempo: + image: grafana/tempo:2.8.2 + user: "0" + command: ["-config.file=/etc/tempo/config.yaml"] + volumes: + - ./infra/otel/tempo/config.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + networks: [data] + restart: unless-stopped + + grafana: + image: grafana/grafana:12.2.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-lab6-local-change-me} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + volumes: + - grafana_data:/var/lib/grafana + - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro + ports: ["127.0.0.1:3000:3000"] + networks: [data, front] + restart: unless-stopped + +networks: + front: + data: + internal: true + +volumes: + pgdata: + redisdata: + s3_data: + s3_config_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + broker_data: \ No newline at end of file diff --git a/evidence/load-lab6.json b/evidence/load-lab6.json new file mode 100644 index 0000000..0c3a916 --- /dev/null +++ b/evidence/load-lab6.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 31.27, + "concurrency": 4, + "successful_requests": 60, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1.919, + "successful_latency_ms": { + "p50": 2063.889, + "p95": 2124.196, + "p99": 2185.684 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/screenshots/file (1).webp b/evidence/screenshots/file (1).webp new file mode 100644 index 0000000..6b0bba4 Binary files /dev/null and b/evidence/screenshots/file (1).webp differ diff --git a/evidence/screenshots/file (2).webp b/evidence/screenshots/file (2).webp new file mode 100644 index 0000000..bce1eaf Binary files /dev/null and b/evidence/screenshots/file (2).webp differ diff --git a/evidence/screenshots/file (3).webp b/evidence/screenshots/file (3).webp new file mode 100644 index 0000000..290fc3a Binary files /dev/null and b/evidence/screenshots/file (3).webp differ diff --git a/evidence/screenshots/file (4).webp b/evidence/screenshots/file (4).webp new file mode 100644 index 0000000..a22fbb4 Binary files /dev/null and b/evidence/screenshots/file (4).webp differ diff --git a/evidence/screenshots/file.webp b/evidence/screenshots/file.webp new file mode 100644 index 0000000..dafce87 Binary files /dev/null and b/evidence/screenshots/file.webp differ diff --git a/evidence/worker-incident.json b/evidence/worker-incident.json new file mode 100644 index 0000000..3d0b891 --- /dev/null +++ b/evidence/worker-incident.json @@ -0,0 +1,12 @@ +{ + "scenario": "stop worker -> 202 queued -> restart -> ready", + "attachment_id": "792f6182-62e6-4f06-9b9b-7c4830ff8fcf", + "status": "ready", + "variants": [ + "crop", + "optimized", + "thumbnail" + ], + "queue_alert_before_worker_start": "firing", + "passed": true +} \ No newline at end of file diff --git a/infra/generate-s3-config.py b/infra/generate-s3-config.py new file mode 100644 index 0000000..2e319f4 --- /dev/null +++ b/infra/generate-s3-config.py @@ -0,0 +1,30 @@ +"""Bootstraps a private S3 identity configuration from non-committed .env. + +The server reads it from a shared internal-only configuration volume. +""" +import json +import os +from pathlib import Path + +bucket = os.environ["S3_BUCKET"] +assert bucket == "messenger-attachments", "Fixed bucket name; change policy and config deliberately" +config = { + "identities": [ + { + "name": "lab4-admin", + "credentials": [{"accessKey": os.environ["S3_ROOT_USER"], + "secretKey": os.environ["S3_ROOT_PASSWORD"]}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + }, + { + "name": "lab4-api", + "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY_ID"], + "secretKey": os.environ["S3_SECRET_ACCESS_KEY"]}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"], + }, + ], +} +target = Path("/s3-config/s3.json") +target.write_text(json.dumps(config, indent=2), encoding="utf-8") +target.chmod(0o644) # Only mounted into S3 service; SeaweedFS runs non-root on some images. +print("PASS: private S3 identity config generated; credentials not logged") diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..d860347 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 512 + +defaults + mode http + log global + option httplog + timeout connect 3s + timeout client 75s + timeout server 75s + timeout check 2s + retries 1 + option redispatch + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold refused 1s + hold timeout 1s + hold other 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server api1 api1:8080 + server api2 api2:8080 diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..2a99cc9 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,62 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 2s + limit_mib: 96 + spike_limit_mib: 24 + batch: + timeout: 3s + send_batch_size: 256 + send_batch_max_size: 512 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s +service: + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + telemetry: + logs: + level: warn diff --git a/infra/otel/grafana/provisioning/dashboards/dashboards.yml b/infra/otel/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..2221158 --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: Messenger Lab 6 + orgId: 1 + folder: MTUCI + type: file + disableDeletion: false + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards/json diff --git a/infra/otel/grafana/provisioning/dashboards/json/dashboard.json b/infra/otel/grafana/provisioning/dashboards/json/dashboard.json new file mode 100644 index 0000000..2cda8fd --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/json/dashboard.json @@ -0,0 +1,481 @@ +{ + "id": null, + "uid": "mtuci-votchitseva-lab6", + "title": "МТУСИ · ЛР6 · Мессенджер", + "description": "API + RabbitMQ worker, OpenTelemetry Collector, Prometheus, Loki and Tempo. All metric labels are low-cardinality.", + "tags": [ + "mtuci", + "lab6", + "observability" + ], + "timezone": "browser", + "schemaVersion": 40, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "API — запросов в секунду", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!=\"/health\"}[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "req/s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "API — доля ошибок 5xx", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!=\"/health\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!=\"/health\"}[1m])), 0.000001)", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "HTTP p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50, sum by(le) (rate(messenger_http_duration_seconds_bucket{route!=\"/health\"}[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "HTTP p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket{route!=\"/health\"}[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "HTTP p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket{route!=\"/health\"}[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Очередь — ожидают обработки", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth)", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Очередь — возраст старейшей", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_age_seconds)", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Worker — jobs/sec", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(outcome)(rate(messenger_jobs_total[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Worker — p95 обработки", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Время до ready — p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + } + }, + { + "id": 11, + "type": "timeseries", + "title": "DB pool — занято", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_db_pool_used{service_name=\"messenger-api\"})", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Worker — загрузка", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_worker_busy{service_name=\"messenger-worker\"}) / clamp_min(max(messenger_worker_capacity{service_name=\"messenger-worker\"}), 1)", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0,211,255,1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + } +} diff --git a/infra/otel/grafana/provisioning/datasources/datasources.yml b/infra/otel/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..7ffd6ce --- /dev/null +++ b/infra/otel/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,30 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: 'trace_id=([0-9a-f]{32})' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + tracesToMetrics: + datasourceUid: prometheus diff --git a/infra/otel/loki/config.yaml b/infra/otel/loki/config.yaml new file mode 100644 index 0000000..6b413aa --- /dev/null +++ b/infra/otel/loki/config.yaml @@ -0,0 +1,27 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h +analytics: + reporting_enabled: false diff --git a/infra/otel/prometheus/alerts.yml b/infra/otel/prometheus/alerts.yml new file mode 100644 index 0000000..165110c --- /dev/null +++ b/infra/otel/prometheus/alerts.yml @@ -0,0 +1,23 @@ +groups: + - name: lab6-signals + interval: 5s + rules: + - alert: MessengerHighErrorRate + expr: | + (sum(rate(messenger_http_requests_total{route!="/health",status_class="5xx"}[1m])) + / clamp_min(sum(rate(messenger_http_requests_total{route!="/health"}[1m])), 0.000001)) > 0.10 + and sum(rate(messenger_http_requests_total{route!="/health"}[1m])) > 0.05 + for: 15s + labels: + severity: warning + annotations: + summary: '5xx responses exceed 10% of business requests' + runbook: 'Check API error rate, Loki errors, Tempo DB spans and postgres health' + - alert: MessengerQueueLagGrowing + expr: max(messenger_queue_oldest_age_seconds) > 20 + for: 15s + labels: + severity: warning + annotations: + summary: 'Oldest queued attachment is older than 20 seconds' + runbook: 'Check worker status, RabbitMQ queue and job failure spans in Tempo' diff --git a/infra/otel/prometheus/prometheus.yml b/infra/otel/prometheus/prometheus.yml new file mode 100644 index 0000000..d251ae1 --- /dev/null +++ b/infra/otel/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/alerts.yml +scrape_configs: + - job_name: collector + static_configs: + - targets: ['otel-collector:8889'] + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/infra/otel/tempo/config.yaml b/infra/otel/tempo/config.yaml new file mode 100644 index 0000000..b7188a1 --- /dev/null +++ b/infra/otel/tempo/config.yaml @@ -0,0 +1,18 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces +compactor: + compaction: + block_retention: 168h diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2b8060d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.115.4 +uvicorn==0.32.0 +sqlalchemy==2.0.36 +asyncpg==0.30.0 +redis==5.2.0 +passlib[bcrypt]==1.7.4 +bcrypt==4.0.1 +python-dotenv==1.0.1 +boto3==1.40.67 +Pillow==11.3.0 +aio-pika==9.4.3 +opentelemetry-api==1.39.1 +opentelemetry-sdk==1.39.1 +opentelemetry-exporter-otlp-proto-http==1.39.1 \ No newline at end of file diff --git a/scripts/check_grade4.py b/scripts/check_grade4.py new file mode 100644 index 0000000..ec63649 --- /dev/null +++ b/scripts/check_grade4.py @@ -0,0 +1,13 @@ +"""Convenience runner. Does not fabricate evidence or control Docker itself.""" +import subprocess +import sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +for command in ( + [sys.executable, str(ROOT / "tests/smoke.py")], + [sys.executable, str(ROOT / "student-tests/run.py")], + [sys.executable, str(ROOT / "student-tests/test_attachments.py")], +): + print("RUN:", " ".join(command), flush=True) + subprocess.run(command, cwd=ROOT, check=True) +print("PASS: public smoke + lab3 concurrency + grade4 negative cases; run extra resilience and memory experiment separately") diff --git a/scripts/experiment_uploads.py b/scripts/experiment_uploads.py new file mode 100644 index 0000000..1dcebb7 --- /dev/null +++ b/scripts/experiment_uploads.py @@ -0,0 +1,71 @@ +"""10 simultaneous real uploads + Docker RSS observations (NO invented metrics).""" +import concurrent.futures +import json +import subprocess +import sys +import threading +import time +from pathlib import Path +from statistics import median +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, arguments, run + + +def mib(value): + value = value.strip() + for suffix, multiplier in (("GiB",1024),("MiB",1),("KiB",1/1024),("B",1/1048576)): + if value.endswith(suffix): + return float(value[:-len(suffix)]) * multiplier + return None + + +def main(): + args = arguments('Concurrent uploads + observed API container memory') + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = client.bootstrap() + file = (ROOT / 'tests/fixtures/sample.png').read_bytes() + results, memory = [], [] + stop = threading.Event() + def observe(): + while not stop.is_set(): + r = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}}|{{.MemUsage}}'], + capture_output=True, text=True, timeout=10) + if r.returncode == 0: + for line in r.stdout.splitlines(): + if '|api1-' in line or '|api2-' in line: + pass + parts = line.split('|') + if len(parts)==2 and ('api1-' in parts[0] or 'api2-' in parts[0]): + amount = mib(parts[1].split('/')[0]) + if amount is not None: + memory.append({'container': parts[0], 'mem_mib': round(amount,3)}) + stop.wait(0.2) + monitor = threading.Thread(target=observe, daemon=True) + monitor.start() + try: + def worker(i): + local = Client(args.base_url,args.ca_file,timeout=30) + start=time.perf_counter() + resp=local.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=run{i}.png', + 202, actor=alice, raw=file, headers={'Content-Type':'image/png'}, schema='Attachment') + return {'elapsed_ms':round((time.perf_counter()-start)*1000,2), 'instance': resp['headers'].get('X-Instance-Id')} + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures=[executor.submit(worker,i) for i in range(10)] + results=[f.result() for f in concurrent.futures.as_completed(futures)] + finally: + stop.set(); monitor.join(timeout=12) + output={'concurrency':10,'file_bytes':len(file),'completed':len(results), + 'median_latency_ms':round(median(x['elapsed_ms'] for x in results),2), + 'api_peak_mem_mib_observed':max([x['mem_mib'] for x in memory],default=None), + 'memory_samples':len(memory),'instances':sorted(set(x['instance'] for x in results))} + target=ROOT / 'evidence' / 'uploads-10.json' + target.write_text(json.dumps(output,ensure_ascii=False,indent=2),encoding='utf-8') + check(len(results)==10, 'not all uploads completed') + print('PASS: 10 parallel uploads; metrics written:',target) + print(json.dumps(output, ensure_ascii=False, indent=2)) + if not memory: + print('WARNING: no Docker memory samples; run docker stats separately for a valid grade-4 experiment') + +if __name__ == '__main__': + run(main) diff --git a/scripts/incident_worker.py b/scripts/incident_worker.py new file mode 100644 index 0000000..d18c27c --- /dev/null +++ b/scripts/incident_worker.py @@ -0,0 +1,95 @@ +import argparse +import base64 +import json +import secrets +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +BASE = 'http://localhost:8080' + + +def call(method, url, body=None, headers=None): + request = urllib.request.Request(url, data=body, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(request, timeout=15) as response: + raw = response.read() + return response.status, json.loads(raw) if raw else {} + except urllib.error.HTTPError as error: + raw = error.read() + return error.code, json.loads(raw) if raw else {} + + +def json_request(method, path, data, auth=None): + headers = {'Content-Type':'application/json'} + if auth: + headers['Authorization'] = 'Bearer '+auth + return call(method, BASE+path, json.dumps(data).encode(), headers) + + +def docker(*args): + subprocess.run(['docker', 'compose', *args], check=True) + + +def alerts(): + try: + code, result = call('GET', 'http://localhost:9090/api/v1/alerts') + if code == 200: + return {a['labels'].get('alertname'):a['state'] for a in result['data']['alerts']} + except Exception: + pass + return {} + + +def main(): + cli = argparse.ArgumentParser() + cli.add_argument('--hold', type=int, default=50) + opts = cli.parse_args() + username = 'obs'+secrets.token_hex(6) + password = secrets.token_urlsafe(24) + status, user = json_request('POST','/api/v1/users',{'username':username,'password':password,'display_name':'Lab6 Test'}) + assert status == 201, user + basic = base64.b64encode((username+':'+password).encode()).decode() + status, auth = call('POST',BASE+'/api/v1/auth/sessions',b'',{'Authorization':'Basic '+basic}) + assert status == 201, auth + token = auth['token'] + status, chat = json_request('POST','/api/v1/chats',{'title':'Lab6 Incident','member_ids':[user['id']]},token) + assert status == 201, chat + docker('stop','worker') + ready = False + try: + sample = Path('tests/fixtures/sample.png').read_bytes() + status, attachment = call('POST', BASE+f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + sample, {'Content-Type':'image/png','Authorization':'Bearer '+token}) + assert status == 202 and attachment['status'] == 'queued', (status,attachment) + attachment_id = attachment['id'] + status, _ = call('GET',BASE+f'/api/v1/attachments/{attachment_id}/content',None,{'Authorization':'Bearer '+token}) + assert status == 409, status + print('202 queued; content 409; worker stopped') + time.sleep(opts.hold) + before = alerts().get('MessengerQueueLagGrowing','not_observed') + print('QueueLag alert:',before) + finally: + docker('start','worker') + deadline = time.monotonic()+100 + while time.monotonic() 202 queued -> restart -> ready', + 'attachment_id':attachment_id,'status':'ready','variants':names, + 'queue_alert_before_worker_start':before,'passed':True} + Path('evidence').mkdir(exist_ok=True) + Path('evidence/worker-incident.json').write_text(json.dumps(result,indent=2),encoding='utf-8') + print('PASS worker recovery:',json.dumps(result,ensure_ascii=False)) + + +if __name__=='__main__': + main() diff --git a/scripts/ops.py b/scripts/ops.py new file mode 100644 index 0000000..344deb1 --- /dev/null +++ b/scripts/ops.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Operator actions for the local Compose stand; volumes are kept.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +SERVICES = ("api1", "api2") +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + + +def compose(*args, capture=False): + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, + text=True, stdout=subprocess.PIPE if capture else None) + return result.stdout.strip() if capture else None + + +def state(service): + cid = compose("ps", "--all", "--quiet", service, capture=True) + if not cid: + raise RuntimeError("No container for " + service) + result = subprocess.check_output( + ["docker", "inspect", "--format", + '{{json .Config.Hostname}} {{json .State.Status}}', cid], text=True) + hostname, status = result.strip().split() + return {"service": service, "container": cid[:12], + "hostname": json.loads(hostname), "status": json.loads(status)} + + +def wait_instances(count=2, timeout=40): + deadline = time.monotonic() + timeout + seen = set() + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=2) as r: + instance = r.headers.get("X-Instance-Id") + if instance: + seen.add(instance) + if len(seen) >= count: + return seen + except (OSError, urllib.error.URLError): + pass + time.sleep(0.2) + raise RuntimeError("Not enough ready API instances through LB") + + +def stop_one(service=None): + target = os.environ.get("TARGET_INSTANCE_ID", "").split(":", 1)[0] + if target: + service = next((s for s in SERVICES if state(s)["hostname"] == target), None) + if not service: + raise RuntimeError("TARGET_INSTANCE_ID is not an API container") + service = service or "api1" + if service not in SERVICES: + raise RuntimeError("Only API replicas may be stopped") + compose("kill", "--signal", "SIGKILL", service) + current = state(service) + if current["status"] != "exited": + raise RuntimeError("API did not stop") + print(json.dumps(current)) + + +def restore(): + compose("up", "-d", "--no-deps", "--no-build", "--wait", "--wait-timeout", "40", *SERVICES) + wait_instances() + + +def restart(): + compose("down") + compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "60") + wait_instances() + + +def rolling_update(): + phases = [] + compose("build", "api1") + # This migration only adds tables and is compatible with the old API. + compose("run", "--rm", "--no-deps", "migrate") + for service in SERVICES: + before = state(service) + started = time.monotonic() + compose("up", "-d", "--no-deps", "--no-build", "--force-recreate", + "--wait", "--wait-timeout", "40", service) + after = state(service) + if before["container"] == after["container"]: + raise RuntimeError("Container was not replaced") + seen = wait_instances() + if not any(i.split(":", 1)[0] == after["hostname"] for i in seen): + raise RuntimeError("LB has not discovered the replacement") + phases.append({"service": service, "started": started, "finished": time.monotonic()}) + print(json.dumps({"replaced": service, "before": before, "after": after})) + return phases + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=["restart", "stop-one", "restore", "rolling-update"]) + parser.add_argument("--service", choices=SERVICES) + args = parser.parse_args() + if args.action == "stop-one": + stop_one(args.service) + else: + {"restart": restart, "restore": restore, "rolling-update": rolling_update}[args.action]() + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..921e224 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +python scripts\ops.py restart +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..da01f20 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/ops.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..8c3cce5 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +python scripts\ops.py stop-one +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..c7b8ec5 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/ops.py stop-one diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..d9e021d --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,24 @@ +"""Entry point: uvicorn as PID 1 with env-driven host/port and graceful shutdown.""" +import os + +import uvicorn + +from .main import app + + +def main() -> None: + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8080")) + uvicorn.run( + app, + host=host, + port=port, + log_level="info", + access_log=True, + timeout_keep_alive=2, + timeout_graceful_shutdown=5, + ) + + +if __name__ == "__main__": + main() diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..b9d77a4 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,29 @@ +import os +import secrets +import redis.asyncio as redis +from passlib.context import CryptContext + +# bcrypt_sha256 avoids bcrypt's 72-byte limit. Old lab2 hashes still verify. +pwd_context = CryptContext(schemes=["bcrypt_sha256", "bcrypt"], deprecated="auto") +CACHE_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0") +SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + +redis_client = redis.from_url(CACHE_URL, decode_responses=True, + socket_connect_timeout=1, socket_timeout=1) + +def hash_password(password: str) -> str: + return pwd_context.hash(password) + +def verify_password(plain_password: str, hashed_password: str) -> bool: + return pwd_context.verify(plain_password, hashed_password) + +async def create_session(user_id: str) -> str: + token = secrets.token_urlsafe(32) + await redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, user_id) + return token + +async def get_user_id_by_token(token: str) -> str | None: + return await redis_client.get(f"session:{token}") + +async def delete_session(token: str) -> None: + await redis_client.delete(f"session:{token}") diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..54d7165 --- /dev/null +++ b/src/db.py @@ -0,0 +1,112 @@ +import os +from datetime import datetime, timezone +from typing import List +from uuid import UUID, uuid4 + +from sqlalchemy import String, ForeignKey, DateTime, Index, Integer, BigInteger +from sqlalchemy.dialects.postgresql import UUID as PG_UUID, JSONB +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql+asyncpg://user:password@localhost:5432/messenger") +engine = create_async_engine( + DATABASE_URL, echo=False, pool_pre_ping=True, pool_size=5, max_overflow=5, + pool_timeout=2, connect_args={"timeout": 1, "command_timeout": 3, + "server_settings": {"statement_timeout": "2500", "lock_timeout": "2000"}}, +) +async_session = async_sessionmaker(engine, expire_on_commit=False) + +class Base(DeclarativeBase): + pass + +class User(Base): + __tablename__ = "users" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + username: Mapped[str] = mapped_column(String(32), unique=True, index=True) + display_name: Mapped[str] = mapped_column(String(100)) + password_hash: Mapped[str] = mapped_column(String(255)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + +class Chat(Base): + __tablename__ = "chats" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + title: Mapped[str] = mapped_column(String(100)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + members: Mapped[List["ChatMember"]] = relationship(back_populates="chat", lazy="selectin") + +class ChatMember(Base): + __tablename__ = "chat_members" + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True) + user_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) + chat: Mapped["Chat"] = relationship(back_populates="members") + user: Mapped["User"] = relationship() + +class Message(Base): + __tablename__ = "messages" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True) + sender_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id")) + text: Mapped[str] = mapped_column(String(4000)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + Index('ix_messages_chat_created_id', 'chat_id', 'created_at', 'id'), + ) + +class IdempotencyKey(Base): + __tablename__ = "idempotency_keys" + user_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"), primary_key=True) + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id"), primary_key=True) + key: Mapped[str] = mapped_column(String(128), primary_key=True) + request_text: Mapped[str] = mapped_column(String(4000)) + response: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + request_attachment_ids: Mapped[list] = mapped_column(JSONB, default=list, nullable=False) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + +class SchemaVersion(Base): + __tablename__ = "schema_versions" + version: Mapped[int] = mapped_column(Integer, primary_key=True) + +class Attachment(Base): + __tablename__ = "attachments" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id"), index=True, nullable=False) + owner_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + object_key: Mapped[str] = mapped_column(String(250), unique=True, nullable=False) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column(String(16), default="queued", nullable=False) + attempts: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False) + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + message_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True) + position: Mapped[int] = mapped_column(Integer, nullable=False) + +# --- НОВЫЕ ТАБЛИЦЫ ДЛЯ 5-Й ЛАБЫ --- +class AttachmentVariant(Base): + __tablename__ = "attachment_variants" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + attachment_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), index=True) + name: Mapped[str] = mapped_column(String(32), nullable=False) + object_key: Mapped[str] = mapped_column(String(250), unique=True, nullable=False) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + width: Mapped[int] = mapped_column(Integer, nullable=False) + height: Mapped[int] = mapped_column(Integer, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + +class Outbox(Base): + __tablename__ = "outbox" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + event_type: Mapped[str] = mapped_column(String(64), nullable=False) + payload: Mapped[dict] = mapped_column(JSONB, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) + processed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) \ No newline at end of file diff --git a/src/images.py b/src/images.py new file mode 100644 index 0000000..a014c54 --- /dev/null +++ b/src/images.py @@ -0,0 +1,112 @@ +"""Validate compressed image bytes without trusting filename or HTTP Content-Type.""" +import os +import threading +import struct +import zlib +import io +import hashlib +from PIL import Image, ImageFile, UnidentifiedImageError, ImageOps +from .storage import MAX_IMAGE_PIXELS + +# Prevent unrestricted decompression of malicious image dimensions. +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS +ImageFile.LOAD_TRUNCATED_IMAGES = False +_IMAGE_LOCK = threading.BoundedSemaphore(1) + +class InvalidImage(Exception): + pass + +class TooManyPixels(Exception): + pass + + +def validate_image_file(file, declared_type: str) -> None: + """Проверяет, что файл является валидным PNG/JPEG и не превышает лимиты.""" + expected = {"image/png": "PNG", "image/jpeg": "JPEG"}.get(declared_type) + if not expected: + raise InvalidImage("unsupported image type") + with _IMAGE_LOCK: + try: + file.seek(0) + with Image.open(file) as probe: + if probe.format != expected: + raise InvalidImage("image MIME does not match its content") + width, height = probe.size + if width <= 0 or height <= 0: + raise InvalidImage("image dimensions are invalid") + if width * height > MAX_IMAGE_PIXELS: + raise TooManyPixels("image dimensions exceed limit") + probe.verify() # verify compressed structure + file.seek(0) + with Image.open(file) as loaded: + loaded.load() # detect damaged truncated payloads + except Image.DecompressionBombError as e: + raise TooManyPixels("image dimensions exceed limit") from e + except Image.DecompressionBombWarning as e: + raise TooManyPixels("image dimensions exceed limit") from e + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError, struct.error, zlib.error) as e: + raise InvalidImage("invalid or truncated PNG/JPEG") from e + finally: + file.seek(0) + + +def make_thumbnail(file_bytes: bytes) -> tuple[bytes, int, int, str]: + """ + Создает JPEG-превью (вписанное в 256x256) из сырых байтов. + Возвращает: (байты_превью, ширина, высота, sha256_превью). + """ + with _IMAGE_LOCK: + with Image.open(io.BytesIO(file_bytes)) as img: + # 1. Применяем EXIF orientation (если картинку сфоткали на телефон боком) + img = ImageOps.exif_transpose(img) + + # 2. Компонуем прозрачность (RGBA, LA) на белый фон + if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): + bg = Image.new("RGB", img.size, (255, 255, 255)) + bg.paste(img, mask=img.convert("RGBA").split()[3]) + img = bg + elif img.mode != "RGB": + # На всякий случай конвертируем все остальные режимы (например, CMYK) в RGB + img = img.convert("RGB") + + # 3. Вписываем в квадрат 256x256 (метод thumbnail сохраняет пропорции и не увеличивает маленькие картинки) + img.thumbnail((256, 256), Image.Resampling.LANCZOS) + + width, height = img.size + + # 4. Сохраняем в JPEG + out_buf = io.BytesIO() + img.save(out_buf, format="JPEG", quality=85) + data = out_buf.getvalue() + + # 5. Считаем хэш для метаданных + sha256 = hashlib.sha256(data).hexdigest() + + return data, width, height, sha256 + +def make_variants(data): + import io + import hashlib + from PIL import Image, ImageOps + thumb_bytes, tw, th, tsha = make_thumbnail(data) + outputs = [('thumbnail', thumb_bytes, 'image/jpeg', tw, th, tsha)] + with _IMAGE_LOCK: + with Image.open(io.BytesIO(data)) as original: + image = ImageOps.exif_transpose(original) + if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info): + rgba = image.convert('RGBA') + canvas = Image.new('RGB', image.size, 'white') + canvas.paste(rgba, mask=rgba.getchannel('A')) + image = canvas + else: + image = image.convert('RGB') + crop = ImageOps.fit(image, (128, 128), Image.Resampling.LANCZOS, centering=(0.5,0.5)) + optimized = image.copy() + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + for name, picture, fmt, quality in [('crop',crop,'JPEG',85), ('optimized',optimized,'WEBP',80)]: + stream = io.BytesIO() + picture.save(stream, format=fmt, quality=quality) + raw = stream.getvalue() + outputs.append((name,raw,'image/webp' if fmt=='WEBP' else 'image/jpeg', + picture.width,picture.height,hashlib.sha256(raw).hexdigest())) + return outputs diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..722895d --- /dev/null +++ b/src/main.py @@ -0,0 +1,600 @@ +from __future__ import annotations +import base64 +import hashlib +from tempfile import SpooledTemporaryFile +from botocore.exceptions import ClientError, BotoCoreError +import asyncio +import binascii +import json +import logging +import os +import re +import socket +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone, timedelta +from typing import Optional + +import aio_pika +from fastapi import Depends, FastAPI, Request, Query +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from pydantic import BaseModel, ConfigDict, Field, field_validator +from sqlalchemy import select, and_, or_, text +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from redis.exceptions import RedisError + +from asyncpg import PostgresError +from .db import engine, async_session, User, Chat, ChatMember, Message, IdempotencyKey, Attachment, MessageAttachment, Outbox, AttachmentVariant +from .auth import hash_password, verify_password, create_session, get_user_id_by_token, delete_session, redis_client, SESSION_TTL_SECONDS +from .storage import s3, bucket, check_storage, upload_original, delete_original, download_original, StorageFailure, MAX_UPLOAD_BYTES +from .images import validate_image_file, InvalidImage, TooManyPixels +from .telemetry import telemetry, trace, inject, extract, SpanKind, Status, StatusCode + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}", re.ASCII) +INSTANCE_ID = socket.gethostname() + ":" + str(os.getpid()) +BROKER_URL = os.getenv("BROKER_URL", "amqp://guest:guest@broker:5672/") +QUEUE_NAME = "attachment.process.v1" + +def now_iso(dt: datetime = None) -> str: + if not dt: + dt = datetime.now(timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + +def error_payload(request_id: str, code: str, message: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str): + self.status = status + self.code = code + self.message = message + super().__init__(message) + +# --- Курсоры для пагинации --- +def _encode_cursor(scope: str, created_at: datetime, obj_id: uuid.UUID) -> str: + raw = f"{scope}|{created_at.isoformat()}|{obj_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + +def _decode_cursor(cursor: str) -> tuple[str, datetime, uuid.UUID]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(padded).decode("utf-8") + scope, dt_str, obj_id_str = raw.split("|", 2) + dt = datetime.fromisoformat(dt_str) + if dt.tzinfo is None: + raise ValueError("timezone required") + return scope, dt, uuid.UUID(obj_id_str) + except Exception: + raise ApiError(400, "invalid_request", "invalid cursor") + +# --- Фоновая задача: Relayer для Outbox --- +async def observe_queue(): + while True: + try: + async with async_session() as session: + row = (await session.execute(text(""" + SELECT count(*) FILTER (WHERE status='queued') AS queued, + coalesce(extract(epoch from now() - min(created_at) + FILTER (WHERE status='queued')), 0) AS age + FROM attachments + """))).one() + telemetry.queue_depth = int(row.queued) + telemetry.queue_age = max(0.0, float(row.age)) + telemetry.db_used = int(engine.pool.checkedout()) + try: + telemetry.db_waiting = len(engine.pool._pool._queue._getters) + except AttributeError: + telemetry.db_waiting = 0 + except Exception: + telemetry.event('queue_metrics_unavailable') + await asyncio.sleep(5) + +async def outbox_relayer(): + while True: + try: + connection = await aio_pika.connect_robust(BROKER_URL) + channel = await connection.channel(publisher_confirms=True) + await channel.declare_queue(QUEUE_NAME, durable=True) + break + except Exception: + telemetry.event('broker_connection_retry') + await asyncio.sleep(5) + + try: + while True: + try: + async with async_session() as session: + records = (await session.scalars( + select(Outbox).where(Outbox.processed_at == None, Outbox.available_at <= datetime.now(timezone.utc)) + .with_for_update(skip_locked=True).limit(10) + )).all() + for record in records: + parent = extract(record.payload) + with telemetry.tracer.start_as_current_span( + 'rabbitmq publish attachment', context=parent, kind=SpanKind.PRODUCER): + carrier = {} + inject(carrier) + await channel.default_exchange.publish( + aio_pika.Message( + body=json.dumps({'attachment_id': record.payload['attachment_id']}).encode(), + headers=carrier, + delivery_mode=aio_pika.DeliveryMode.PERSISTENT), + routing_key=QUEUE_NAME, mandatory=True) + telemetry.event('job_published', job_type='attachment') + record.processed_at = datetime.now(timezone.utc) + await session.commit() + except Exception: + telemetry.event('outbox_publish_retry') + await asyncio.sleep(2) + await asyncio.sleep(0.5) + finally: + await connection.close() + +@asynccontextmanager +async def lifespan(app: FastAPI): + tasks = [asyncio.create_task(outbox_relayer()), asyncio.create_task(observe_queue())] + yield + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + await redis_client.aclose() + await engine.dispose() + telemetry.shutdown() + +app = FastAPI(title="Messenger", version="1.6.0", lifespan=lifespan) +security_bearer = HTTPBearer(auto_error=False) + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request_id = request.headers.get('X-Request-Id') or str(uuid.uuid4()) + request.state.request_id = request_id + start = __import__('time').monotonic() + remote_context = extract(request.headers) + method = request.method + with telemetry.tracer.start_as_current_span( + 'HTTP ' + method, context=remote_context, kind=SpanKind.SERVER) as span: + try: + response = await call_next(request) + except Exception: + response = JSONResponse(status_code=500, content=error_payload(request_id, 'internal_error', 'internal error')) + response.headers['X-Request-Id'] = request_id + response.headers['X-Instance-Id'] = INSTANCE_ID + route_obj = request.scope.get('route') + route = getattr(route_obj, 'path', '/unmatched') + if route.startswith('/health/'): + route = '/health' + status_class = str(response.status_code // 100) + 'xx' + attrs = {'method':method,'route':route,'status_class':status_class} + span.set_attribute('http.request.method', method) + span.set_attribute('http.route', route) + span.set_attribute('http.response.status_code', response.status_code) + if response.status_code >= 500: + span.set_status(Status(StatusCode.ERROR)) + telemetry.requests.add(1, attrs) + telemetry.http_duration.record(__import__('time').monotonic()-start, attrs) + telemetry.event('http_request', method=method, route=route, + status=response.status_code, request_id=request_id) + return response + +def _rid(request: Request) -> str: + return getattr(request.state, "request_id", None) or str(uuid.uuid4()) + +# --- Обработчики ошибок --- +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + headers = {} + if exc.status == 401: + if request.url.path == "/api/v1/auth/sessions": + headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"' + else: + headers["WWW-Authenticate"] = 'Bearer realm="messenger"' + return JSONResponse(status_code=exc.status, content=error_payload(_rid(request), exc.code, exc.message), headers=headers) + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + return JSONResponse(status_code=400, content=error_payload(_rid(request), "invalid_request", "invalid request")) + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + return JSONResponse(status_code=exc.status_code, content=error_payload(_rid(request), "not_found", str(exc.detail))) + +@app.exception_handler(InvalidImage) +async def invalid_image_handler(request: Request, exc: InvalidImage): + return JSONResponse(status_code=415, content=error_payload(_rid(request), "unsupported_media_type", str(exc))) + +@app.exception_handler(TooManyPixels) +async def too_large_image_handler(request: Request, exc: TooManyPixels): + return JSONResponse(status_code=413, content=error_payload(_rid(request), "payload_too_large", str(exc))) + +@app.exception_handler(RedisError) +@app.exception_handler(SQLAlchemyError) +@app.exception_handler(PostgresError) +@app.exception_handler(OSError) +@app.exception_handler(TimeoutError) +@app.exception_handler(ClientError) +@app.exception_handler(BotoCoreError) +@app.exception_handler(StorageFailure) +async def dependency_error_handler(request: Request, exc: Exception): + return JSONResponse(status_code=503, content=error_payload(_rid(request), "unavailable", "dependency unavailable")) + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + return JSONResponse(status_code=500, content=error_payload(_rid(request), "internal_error", "internal error")) + +# --- Схемы --- +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(strict=True) + display_name: str = Field(strict=True, min_length=1, max_length=100) + password: str = Field(strict=True, min_length=12, max_length=128) + + @field_validator("display_name", "password") + @classmethod + def not_blank(cls, value): + if not value.strip(): + raise ValueError("blank value") + return value + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(strict=True, min_length=1, max_length=100) + member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def not_blank(cls, value): + if not value.strip(): + raise ValueError("blank title") + return value + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(strict=True, max_length=4000) + attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def unique_attachments(cls, value): + if len(set(value)) != len(value): + raise ValueError("duplicate attachment id") + return value + +async def get_actor(credentials: HTTPAuthorizationCredentials = Depends(security_bearer)) -> dict: + if not credentials: + raise ApiError(401, "unauthorized", "missing token") + + user_id_str = await get_user_id_by_token(credentials.credentials) + if not user_id_str: + raise ApiError(401, "unauthorized", "invalid or expired token") + + async with async_session() as session: + user = await session.get(User, uuid.UUID(user_id_str)) + if not user: + raise ApiError(401, "unauthorized", "user not found") + return {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)} + +async def require_chat_member(session, chat_id: uuid.UUID, user_id: str): + member = await session.scalar(select(ChatMember).where(and_( + ChatMember.chat_id == chat_id, ChatMember.user_id == uuid.UUID(user_id)))) + if not member: + raise ApiError(404, "not_found", "chat not found") + +@app.get("/health/live") +def liveness(): + return {"status": "ok"} + +@app.get("/health/ready") +async def readiness(): + async with asyncio.timeout(1.5): + async with async_session() as session: + version = await session.scalar(text("SELECT version FROM schema_versions WHERE version = 6")) + if version != 6: + raise ApiError(503, "unavailable", "migration required") + await redis_client.ping() + await run_in_threadpool(check_storage) + return {"status": "ok"} + +@app.post("/api/v1/users", status_code=201) +async def register_user(payload: CreateUser): + if not USERNAME_RE.fullmatch(payload.username): + raise ApiError(400, "invalid_request", "invalid username") + async with async_session() as session: + existing = await session.scalar(select(User).where(User.username == payload.username)) + if existing: + raise ApiError(409, "conflict", "username already exists") + password_hash = await run_in_threadpool(hash_password, payload.password) + user = User(username=payload.username, display_name=payload.display_name, password_hash=password_hash) + session.add(user) + try: + await session.commit() + except IntegrityError: + await session.rollback() + raise ApiError(409, "conflict", "username already exists") + return {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)} + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session_endpoint(request: Request): + try: + scheme, encoded = request.headers.get("Authorization", "").split(" ", 1) + if scheme.lower() != "basic": + raise ValueError("Basic required") + username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "invalid credentials") + + async with async_session() as session: + user = await session.scalar(select(User).where(User.username == username)) + if not user or not await run_in_threadpool(verify_password, password, user.password_hash): + raise ApiError(401, "unauthorized", "invalid credentials") + token = await create_session(str(user.id)) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + return { + "token": token, "token_type": "Bearer", "expires_at": now_iso(expires_at), + "user": {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)} + } + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(credentials: HTTPAuthorizationCredentials = Depends(security_bearer)): + if not credentials: + raise ApiError(401, "unauthorized", "missing token") + user_id = await get_user_id_by_token(credentials.credentials) + if not user_id: + raise ApiError(401, "unauthorized", "invalid or expired token") + await delete_session(credentials.credentials) + +@app.get("/api/v1/users/me") +async def current_user(actor: dict = Depends(get_actor)): + return actor + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(payload: CreateChat, actor: dict = Depends(get_actor)): + member_ids = [str(u) for u in payload.member_ids] + if actor["id"] not in member_ids: + raise ApiError(400, "invalid_request", "creator must be a member") + if len(set(member_ids)) != len(member_ids): + raise ApiError(400, "invalid_request", "duplicate members") + async with async_session() as session: + users = await session.scalars(select(User.id).where(User.id.in_([uuid.UUID(m) for m in member_ids]))) + if len(list(users)) != len(set(member_ids)): + raise ApiError(400, "invalid_request", "unknown member") + chat = Chat(title=payload.title) + session.add(chat) + await session.flush() + for mid in member_ids: + session.add(ChatMember(chat_id=chat.id, user_id=uuid.UUID(mid))) + await session.commit() + return {"id": str(chat.id), "title": chat.title, "member_ids": member_ids, "created_at": now_iso(chat.created_at)} + +@app.get("/api/v1/chats") +async def list_chats(limit: int = Query(50, ge=1, le=100), cursor: Optional[str] = None, actor: dict = Depends(get_actor)): + async with async_session() as session: + stmt = select(Chat).join(ChatMember).where(ChatMember.user_id == uuid.UUID(actor["id"])) + if cursor: + scope, c_dt, c_id = _decode_cursor(cursor) + if scope != "chats": + raise ApiError(400, "invalid_request", "invalid cursor scope") + stmt = stmt.where(or_(Chat.created_at > c_dt, and_(Chat.created_at == c_dt, Chat.id > c_id))) + stmt = stmt.order_by(Chat.created_at.asc(), Chat.id.asc()).limit(limit + 1) + chats = (await session.scalars(stmt)).unique().all() + has_more = len(chats) > limit + page = chats[:limit] + next_cursor = _encode_cursor("chats", page[-1].created_at, page[-1].id) if has_more and page else None + items = [{"id": str(c.id), "title": c.title, "created_at": now_iso(c.created_at), "member_ids": [str(m.user_id) for m in c.members]} for c in page] + return {"items": items, "next_cursor": next_cursor} + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: uuid.UUID, actor: dict = Depends(get_actor)): + async with async_session() as session: + chat = await session.scalar(select(Chat).join(ChatMember).where(Chat.id == chat_id, ChatMember.user_id == uuid.UUID(actor["id"]))) + if not chat: + raise ApiError(404, "not_found", "chat not found") + return {"id": str(chat.id), "title": chat.title, "member_ids": [str(m.user_id) for m in chat.members], "created_at": now_iso(chat.created_at)} + +def attachment_json(item: Attachment, variants: list[AttachmentVariant] = None) -> dict: + return { + "id": str(item.id), "chat_id": str(item.chat_id), + "owner_id": str(item.owner_id), "filename": item.filename, + "content_type": item.content_type, "size_bytes": item.size_bytes, + "sha256": item.sha256, "status": item.status, + "variants": [{ + "name": v.name, "content_type": v.content_type, "size_bytes": v.size_bytes, + "width": v.width, "height": v.height, "sha256": v.sha256 + } for v in (variants or [])], + "error_code": item.error_code, + "created_at": now_iso(item.created_at), "updated_at": now_iso(item.updated_at), + } + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment( + chat_id: uuid.UUID, request: Request, filename: str = Query(...), + actor: dict = Depends(get_actor), +): + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + + if not (1 <= len(filename) <= 255) or any(c in filename for c in ("/", "\\")) or any(ord(c) < 32 or ord(c) == 127 for c in filename): + raise ApiError(400, "invalid_request", "invalid filename") + + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "only PNG and JPEG are accepted") + + hasher = hashlib.sha256() + size = 0 + with SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as original: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "image exceeds 10 MiB") + hasher.update(chunk) + original.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "empty image") + await run_in_threadpool(validate_image_file, original, declared_type) + + item_id = uuid.uuid4() + key = f"originals/{item_id.hex}/{uuid.uuid4().hex}" + with telemetry.tracer.start_as_current_span('s3 stage original', kind=SpanKind.CLIENT): + await run_in_threadpool(upload_original, original, key, size, declared_type) + try: + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + item = Attachment( + id=item_id, chat_id=chat_id, owner_id=uuid.UUID(actor["id"]), + filename=filename, object_key=key, content_type=declared_type, + size_bytes=size, sha256=hasher.hexdigest(), status="queued", + ) + session.add(item) + # Добавляем задачу в Outbox + carrier = {} + inject(carrier) + payload = {'attachment_id': str(item_id), **carrier} + session.add(Outbox(event_type='attachment.process.v1', payload=payload)) + with telemetry.tracer.start_as_current_span('db commit accepted job', kind=SpanKind.CLIENT): + await session.commit() + telemetry.event('attachment_accepted', job_type='attachment') + return attachment_json(item) + except Exception: + try: + await run_in_threadpool(delete_original, key) + except Exception: + logging.getLogger(__name__).warning("S3 compensation failed; reconcile needed") + raise + +@app.get("/api/v1/attachments/{attachment_id}") +async def attachment_metadata(attachment_id: uuid.UUID, actor: dict = Depends(get_actor)): + async with async_session() as session: + item = await session.get(Attachment, attachment_id) + if item is None: + raise ApiError(404, "not_found", "attachment not found") + await require_chat_member(session, item.chat_id, actor["id"]) + + variants = (await session.scalars(select(AttachmentVariant).where(AttachmentVariant.attachment_id == item.id))).all() + return attachment_json(item, variants) + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def attachment_bytes( + attachment_id: uuid.UUID, variant: str = "original", actor: dict = Depends(get_actor) +): + async with async_session() as session: + item = await session.get(Attachment, attachment_id) + if item is None: + raise ApiError(404, "not_found", "attachment not found") + await require_chat_member(session, item.chat_id, actor["id"]) + + if item.status != "ready": + raise ApiError(409, "conflict", "attachment is not ready") + + if variant == "original": + key, content_type, size = item.object_key, item.content_type, item.size_bytes + else: + var = await session.scalar(select(AttachmentVariant).where( + AttachmentVariant.attachment_id == item.id, AttachmentVariant.name == variant + )) + if not var: + raise ApiError(404, "not_found", "variant not found") + key, content_type, size = var.object_key, var.content_type, var.size_bytes + + body = await run_in_threadpool(download_original, key) + def chunks(): + try: + yield from body.iter_chunks(chunk_size=64 * 1024) + finally: + body.close() + + return StreamingResponse(chunks(), media_type=content_type, + headers={"Content-Length": str(size), "Cache-Control": "private, no-store"}) + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def post_message(chat_id: uuid.UUID, payload: CreateMessage, request: Request, actor: dict = Depends(get_actor)): + key = request.headers.get("Idempotency-Key", "") + if not KEY_RE.fullmatch(key): + raise ApiError(400, "invalid_request", "invalid Idempotency-Key") + if not payload.text.strip() and not payload.attachment_ids: + raise ApiError(400, "invalid_request", "empty message") + attachment_ids = [str(a) for a in payload.attachment_ids] + + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + + if attachment_ids: + attachments = (await session.scalars(select(Attachment).where( + Attachment.id.in_(payload.attachment_ids), Attachment.chat_id == chat_id + ))).all() + if len(attachments) != len(attachment_ids): + raise ApiError(404, "not_found", "attachment not found") + # В 5-й лабе можно прикреплять файлы в статусе queued/processing, но не failed + if any(a.status == "failed" for a in attachments): + raise ApiError(409, "conflict", "attachment failed and cannot be attached") + + now = datetime.now(timezone.utc) + scope = {"user_id": uuid.UUID(actor["id"]), "chat_id": chat_id, "key": key} + await session.execute(insert(IdempotencyKey).values( + **scope, request_text=payload.text, request_attachment_ids=attachment_ids, + expires_at=now + timedelta(hours=24) + ).on_conflict_do_nothing()) + record = await session.scalar(select(IdempotencyKey).where( + IdempotencyKey.user_id == scope["user_id"], + IdempotencyKey.chat_id == chat_id, IdempotencyKey.key == key + ).with_for_update()) + + if record.response is not None and record.expires_at > now: + if record.request_text != payload.text or record.request_attachment_ids != attachment_ids: + raise ApiError(409, "conflict", "key used with another message") + return record.response + + msg = Message(chat_id=chat_id, sender_id=uuid.UUID(actor["id"]), text=payload.text) + session.add(msg) + await session.flush() + for pos, a_id in enumerate(payload.attachment_ids): + session.add(MessageAttachment(message_id=msg.id, attachment_id=a_id, position=pos)) + + result = { + "id": str(msg.id), "chat_id": str(msg.chat_id), "sender_id": str(msg.sender_id), + "text": msg.text, "created_at": now_iso(msg.created_at), + "attachment_ids": attachment_ids, + } + record.request_text = payload.text + record.request_attachment_ids = attachment_ids + record.expires_at = now + timedelta(hours=24) + record.response = result + await session.commit() + return result + +@app.get("/api/v1/chats/{chat_id}/messages") +async def get_messages(chat_id: uuid.UUID, limit: int = Query(50, ge=1, le=100), cursor: Optional[str] = None, actor: dict = Depends(get_actor)): + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + stmt = select(Message).where(Message.chat_id == chat_id) + if cursor: + scope, c_dt, c_id = _decode_cursor(cursor) + if scope != str(chat_id): + raise ApiError(400, "invalid_request", "invalid cursor scope") + stmt = stmt.where(or_(Message.created_at > c_dt, and_(Message.created_at == c_dt, Message.id > c_id))) + stmt = stmt.order_by(Message.created_at.asc(), Message.id.asc()).limit(limit + 1) + messages = (await session.scalars(stmt)).all() + has_more = len(messages) > limit + page = messages[:limit] + next_cursor = _encode_cursor(str(chat_id), page[-1].created_at, page[-1].id) if has_more and page else None + + msg_ids = [m.id for m in page] + links = [] + if msg_ids: + links = (await session.execute(select(MessageAttachment).where( + MessageAttachment.message_id.in_(msg_ids) + ).order_by(MessageAttachment.message_id, MessageAttachment.position))).scalars().all() + linked: dict[uuid.UUID, list[str]] = {m.id: [] for m in page} + for link in links: + linked[link.message_id].append(str(link.attachment_id)) + items = [{ + "id": str(m.id), "chat_id": str(m.chat_id), "sender_id": str(m.sender_id), + "text": m.text, "created_at": now_iso(m.created_at), + "attachment_ids": linked[m.id], + } for m in page] + return {"items": items, "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..db29dfb --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,25 @@ +import asyncio + +from sqlalchemy import text +from .db import Base, engine + + +async def main(): + async with engine.begin() as conn: + await conn.execute(text("SELECT pg_advisory_xact_lock(300300)")) + await conn.run_sync(Base.metadata.create_all) + + await conn.execute(text("ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code VARCHAR(64)")) + + await conn.execute(text("INSERT INTO schema_versions(version) VALUES (3) ON CONFLICT DO NOTHING")) + await conn.execute(text("INSERT INTO schema_versions(version) VALUES (4) ON CONFLICT DO NOTHING")) + await conn.execute(text("INSERT INTO schema_versions(version) VALUES (5) ON CONFLICT DO NOTHING")) + await conn.execute(text("ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempts INTEGER NOT NULL DEFAULT 0")) + await conn.execute(text("ALTER TABLE outbox ADD COLUMN IF NOT EXISTS available_at TIMESTAMPTZ NOT NULL DEFAULT now()")) + await conn.execute(text("INSERT INTO schema_versions(version) VALUES (6) ON CONFLICT DO NOTHING")) + await engine.dispose() + print("Migration 5 applied: outbox, variants and async processing added") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/src/private_check.py b/src/private_check.py new file mode 100644 index 0000000..72d04d6 --- /dev/null +++ b/src/private_check.py @@ -0,0 +1,22 @@ +"""Prove S3 is not public and API-scoped credentials cannot list other buckets.""" +import urllib.error +import urllib.request +from botocore.exceptions import ClientError +from .storage import s3, bucket +import os +import uuid + +url = os.getenv("S3_ENDPOINT", "http://s3:8333").rstrip("/") + "/" + bucket + "/random-inaccessible-key" +try: + urllib.request.urlopen(url, timeout=4) +except urllib.error.HTTPError as exc: + assert exc.code in (401, 403), f"expected S3 denied, got {exc.code}" +else: + raise AssertionError("UNSAFE: direct unsigned S3 request succeeded") +try: + s3.create_bucket(Bucket="forbidden-probe-" + uuid.uuid4().hex[:16]) +except ClientError as exc: + assert exc.response["Error"]["Code"] in ("AccessDenied", "403"), "unexpected S3 result" +else: + raise AssertionError("UNSAFE: API credentials have global create-bucket privileges") +print("PASS: S3 requires authentication; API credentials restricted to own bucket") diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..a5f9e9d --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,51 @@ +"""Safe orphan S3 cleanup (dry run default). Use inside api container. + +A crash between S3 PUT and PostgreSQL COMMIT can leave an unreferenced object. +Only objects older than the grace period are candidates. No deletion if DB read fails. +""" +import argparse +import asyncio +from datetime import datetime, timezone, timedelta +from sqlalchemy import select +from .db import async_session, Attachment, engine +from .storage import s3, bucket +from starlette.concurrency import run_in_threadpool + + +def list_s3(): + keys = [] + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix="originals/"): + keys.extend((i["Key"], i["LastModified"]) for i in page.get("Contents", [])) + return keys + + +async def run(apply=False, grace_seconds=3600): + if grace_seconds < 3600: + raise ValueError("grace must be at least 3600 seconds") + # Load DB references before enumerating/deleting S3 objects. Fail closed. + async with async_session() as session: + referenced = set((await session.scalars(select(Attachment.object_key))).all()) + objects = await run_in_threadpool(list_s3) + threshold = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + candidates = [key for key, when in objects if key not in referenced and when < threshold] + print(f"referenced={len(referenced)} objects={len(objects)} old_orphans={len(candidates)} apply={apply}") + for key in candidates: + # Avoid listing filenames/secrets: keys are random IDs, not user inputs. + if apply: + await run_in_threadpool(s3.delete_object, Bucket=bucket, Key=key) + print(("deleted " if apply else "would_delete ") + key) + return len(candidates) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="really delete aged orphan objects") + parser.add_argument("--grace-seconds", type=int, default=3600) + args = parser.parse_args() + async def main(): + try: + await run(args.apply, args.grace_seconds) + finally: + await engine.dispose() + asyncio.run(main()) diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..45ba7d8 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,43 @@ +"""S3-compatible private storage. Only API-scoped credentials are available here.""" +import os +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, BotoCoreError + +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +bucket = os.getenv("S3_BUCKET", "messenger-attachments") +s3 = boto3.client( + "s3", endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name=os.getenv("S3_REGION", "us-east-1"), + aws_access_key_id=os.environ.get("S3_ACCESS_KEY_ID", "unset"), + aws_secret_access_key=os.environ.get("S3_SECRET_ACCESS_KEY", "unset"), + config=Config(s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=30, + retries={"mode": "standard", "max_attempts": 2}), +) + +class StorageFailure(Exception): + pass + + +def check_storage(): + # Scoped user has ListBucket but no privileges outside this bucket. + s3.list_objects_v2(Bucket=bucket, MaxKeys=1) + + +def upload_original(file, key, size, content_type): + file.seek(0) + s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type) + + +def delete_original(key): + s3.delete_object(Bucket=bucket, Key=key) + + +def download_original(key): + try: + return s3.get_object(Bucket=bucket, Key=key)["Body"] + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageFailure("Object missing") from None + raise diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..6e5e7e5 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,87 @@ +import logging +import os +import socket +import time + +from opentelemetry import metrics, trace +from opentelemetry.propagate import extract, inject +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + +BASE = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/') +SERVICE = os.getenv('OTEL_SERVICE_NAME', 'messenger-api') + +class Telemetry: + def __init__(self): + resource = Resource.create({ + 'service.name': SERVICE, + 'service.version': '1.6.0', + 'service.instance.id': os.getenv('INSTANCE_ID', socket.gethostname()), + 'deployment.environment.name': 'lab', + }) + self.trace_provider = TracerProvider(resource=resource) + self.trace_provider.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=BASE+'/v1/traces', timeout=2), + max_queue_size=256, max_export_batch_size=32, + schedule_delay_millis=2000, export_timeout_millis=2000)) + trace.set_tracer_provider(self.trace_provider) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=BASE+'/v1/metrics', timeout=2), + export_interval_millis=5000, export_timeout_millis=3000) + self.metric_provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(self.metric_provider) + self.log_provider = LoggerProvider(resource=resource) + self.log_provider.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=BASE+'/v1/logs', timeout=2), + max_queue_size=256, max_export_batch_size=32, schedule_delay_millis=2000)) + self.log = logging.getLogger('messenger.telemetry') + self.log.setLevel(logging.INFO) + self.log.propagate = False + self.log.addHandler(LoggingHandler(level=logging.INFO, logger_provider=self.log_provider)) + self.tracer = trace.get_tracer('messenger', '1.6.0') + meter = metrics.get_meter('messenger', '1.6.0') + self.requests = meter.create_counter('messenger_http_requests', unit='1') + self.http_duration = meter.create_histogram('messenger_http_duration', unit='s', explicit_bucket_boundaries_advisory=[0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.4, 0.8, 1.5, 3.0, 6.0, 12.0]) + self.jobs = meter.create_counter('messenger_jobs', unit='1') + self.job_duration = meter.create_histogram('messenger_job_duration', unit='s', explicit_bucket_boundaries_advisory=[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0]) + self.time_to_ready = meter.create_histogram('messenger_time_to_ready', unit='s', explicit_bucket_boundaries_advisory=[0.1, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300]) + self.queue_depth = 0 + self.queue_age = 0.0 + self.db_used = 0 + self.db_waiting = 0 + self.worker_busy = 0 + self.worker_capacity = 1 if SERVICE == 'messenger-worker' else 0 + self.queue_metric = meter.create_observable_gauge('messenger_queue_depth', callbacks=[lambda _: [metrics.Observation(self.queue_depth, {'queue':'attachment.process.v1'})],], unit='1') + self.age_metric = meter.create_observable_gauge('messenger_queue_oldest_age', callbacks=[lambda _: [metrics.Observation(self.queue_age, {'queue':'attachment.process.v1'})],], unit='s') + self.used_metric = meter.create_observable_gauge('messenger_db_pool_used', callbacks=[lambda _: [metrics.Observation(self.db_used)],], unit='1') + self.waiting_metric = meter.create_observable_gauge('messenger_db_pool_waiting', callbacks=[lambda _: [metrics.Observation(self.db_waiting)],], unit='1') + self.busy_metric = meter.create_observable_gauge('messenger_worker_busy', callbacks=[lambda _: [metrics.Observation(self.worker_busy)],], unit='1') + self.capacity_metric = meter.create_observable_gauge('messenger_worker_capacity', callbacks=[lambda _: [metrics.Observation(self.worker_capacity)],], unit='1') + + def event(self, name, **safe_fields): + payload = 'event=' + name + for key, value in safe_fields.items(): + if key in ('route', 'method', 'status', 'outcome', 'job_type', 'request_id'): + payload += ' ' + key + '=' + str(value).replace(' ', '_')[:100] + span_context = trace.get_current_span().get_span_context() + if span_context.is_valid: + payload += ' trace_id=%032x span_id=%016x' % (span_context.trace_id, span_context.span_id) + self.log.info(payload) + + def shutdown(self): + self.trace_provider.shutdown() + self.metric_provider.shutdown() + self.log_provider.shutdown() + +telemetry = Telemetry() + +__all__ = ['telemetry', 'trace', 'extract', 'inject', 'SpanKind', 'Status', 'StatusCode'] diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..f7b8451 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,122 @@ +import asyncio +import io +import json +import logging +import os +import time +import uuid +from datetime import datetime, timezone, timedelta + +import aio_pika +from PIL import Image, ImageOps +from sqlalchemy import select + +from .db import async_session, engine, Attachment, AttachmentVariant, Outbox +from .storage import upload_original, download_original +from .images import make_thumbnail, make_variants +from .telemetry import telemetry, extract, SpanKind, Status, StatusCode + +BROKER_URL = os.getenv('BROKER_URL', 'amqp://guest:guest@broker:5672/') +QUEUE_NAME = 'attachment.process.v1' +MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5')) + +async def process_attachment(message: aio_pika.IncomingMessage): + ctx = extract(message.headers or {}) + started = time.monotonic() + telemetry.worker_busy = 1 + outcome = 'success' + try: + with telemetry.tracer.start_as_current_span( + 'attachment.process', context=ctx, kind=SpanKind.CONSUMER) as span: + async with message.process(requeue=True): + payload = json.loads(message.body.decode()) + attachment_id = uuid.UUID(payload['attachment_id']) + telemetry.event('job_started', job_type='attachment') + async with async_session() as session: + att = await session.get(Attachment, attachment_id, with_for_update=True) + if not att or att.status in ('ready', 'failed'): + telemetry.event('job_duplicate_skipped', job_type='attachment') + return + att.status = 'processing' + att.attempts = int(att.attempts or 0) + 1 + attempt = att.attempts + await session.commit() + try: + with telemetry.tracer.start_as_current_span('s3.download_original', kind=SpanKind.CLIENT): + stream = await asyncio.to_thread(download_original, att.object_key) + data = await asyncio.to_thread(stream.read) + await asyncio.to_thread(stream.close) + with telemetry.tracer.start_as_current_span('image.create_variants', kind=SpanKind.INTERNAL): + outputs = await asyncio.to_thread(make_variants, data) + with telemetry.tracer.start_as_current_span('s3.store_variants', kind=SpanKind.CLIENT): + for name, raw, mime, w, h, sha in outputs: + key = f'variants/{attachment_id.hex}/{name}' + await asyncio.to_thread(upload_original, io.BytesIO(raw), key, len(raw), mime) + with telemetry.tracer.start_as_current_span('db.commit_ready', kind=SpanKind.CLIENT): + async with async_session() as session: + locked = await session.get(Attachment, attachment_id, with_for_update=True) + if locked and locked.status != 'ready': + for name, raw, mime, w, h, sha in outputs: + previous = await session.scalar(select(AttachmentVariant).where( + AttachmentVariant.attachment_id == attachment_id, + AttachmentVariant.name == name)) + if previous is None: + previous = AttachmentVariant(attachment_id=attachment_id, name=name) + session.add(previous) + previous.object_key = f'variants/{attachment_id.hex}/{name}' + previous.content_type = mime + previous.size_bytes = len(raw) + previous.width = w + previous.height = h + previous.sha256 = sha + locked.status = 'ready' + locked.error_code = None + await session.commit() + elapsed = (datetime.now(timezone.utc) - locked.created_at).total_seconds() + telemetry.time_to_ready.record(max(0.0,elapsed), {'job_type':'attachment'}) + telemetry.event('job_ready', job_type='attachment', outcome='success') + except Exception as exc: + outcome = 'failed' + span.set_status(Status(StatusCode.ERROR)) + async with async_session() as session: + locked = await session.get(Attachment, attachment_id, with_for_update=True) + if locked and locked.status != 'ready': + if attempt >= MAX_ATTEMPTS or isinstance(exc, (ValueError, Image.DecompressionBombError)): + locked.status = 'failed' + locked.error_code = 'retry_exhausted' if attempt >= MAX_ATTEMPTS else 'invalid_image' + telemetry.event('job_failed', job_type='attachment', outcome='permanent') + else: + locked.status = 'queued' + locked.error_code = None + delay = min(60, 2 ** min(attempt, 5)) + session.add(Outbox( + event_type='attachment.process.v1', + payload={'attachment_id':str(attachment_id), **dict(message.headers or {})}, + available_at=datetime.now(timezone.utc)+timedelta(seconds=delay))) + telemetry.event('job_retry_scheduled', job_type='attachment', outcome='transient') + await session.commit() + except Exception: + outcome = 'failed' + telemetry.event('job_delivery_exception', job_type='attachment') + raise + finally: + telemetry.jobs.add(1, {'job_type':'attachment', 'outcome':outcome}) + telemetry.job_duration.record(time.monotonic()-started, {'job_type':'attachment','outcome':outcome}) + telemetry.worker_busy = 0 + +async def main(): + connection = await aio_pika.connect_robust(BROKER_URL) + channel = await connection.channel(publisher_confirms=True) + queue = await channel.declare_queue(QUEUE_NAME, durable=True) + await channel.set_qos(prefetch_count=1) + await queue.consume(process_attachment) + telemetry.event('worker_started', job_type='attachment') + try: + await asyncio.Future() + finally: + await connection.close() + await engine.dispose() + telemetry.shutdown() + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 0000000..588ebe2 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import sys +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Lab3: concurrent idempotency and validation") + c = Client(args.base_url, args.ca_file) + c.call("POST", "/api/v1/users", 400, + body={"username": "valid\n", "display_name": "Test", "password": "LongPassword123"}) + registration = {"username": "race_" + uuid.uuid4().hex[:12], + "display_name": "Race", "password": "LongPassword123"} + registration_barrier = threading.Barrier(2) + + def register(_): + local = Client(args.base_url, args.ca_file) + registration_barrier.wait() + return local.call("POST", "/api/v1/users", [201, 409], + body=registration, schema="User")["status"] + + with ThreadPoolExecutor(max_workers=2) as pool: + check(sorted(pool.map(register, range(2))) == [201, 409], "registration race") + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + key = uuid.uuid4().hex + barrier = threading.Barrier(12) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.message(alice, chat, "Одна запись", key=key) + + with ThreadPoolExecutor(max_workers=12) as pool: + responses = list(pool.map(send, range(12))) + first = responses[0]["json"] + check(all(r["json"] == first for r in responses), "concurrent replay differs") + instances = {r["headers"]["X-Instance-Id"] for r in responses} + check(len(instances) >= 2, "same key must reach at least two replicas") + page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(page["items"] == [first], "concurrent retry created duplicates") + c.call("POST", path, 409, actor=alice, body={"text": "Изменённый текст"}, + headers={"Idempotency-Key": key}) + c.call("POST", path, 404, actor=eve, body={"text": "Одна запись"}, + headers={"Idempotency-Key": key}) + # The same key belongs to a different scope for Bob and for another chat. + other = c.message(bob, chat, "Другая область", key=key)["json"] + check(other["id"] != first["id"], "different users share an idempotency scope") + second_chat = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Второй", "member_ids": [alice["id"]]}, + schema="Chat")["json"] + other = c.message(alice, second_chat, "Одна запись", key=key)["json"] + check(other["id"] != first["id"], "different chats share an idempotency scope") + for bad_key in ("", "bad key", "x" * 129): + c.call("POST", path, 400, actor=alice, body={"text": "Текст"}, + headers={"Idempotency-Key": bad_key}) + for body in ({"text": " "}, {"text": 123}, {"text": "x", "sender_id": alice["id"]}): + c.call("POST", path, 400, actor=alice, body=body, + headers={"Idempotency-Key": uuid.uuid4().hex}) + c.call("POST", path, 400, actor=alice, raw=b'{"text":', + headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call("POST", path, 401, actor=alice, body={"text": "Одна запись"}, + headers={"Idempotency-Key": key}) + print("PASS: 12 concurrent requests across %d replicas, one message" % len(instances)) + print("PASS: conflict, key scope, validation, access and revoked session") + print("PASS: concurrent registration, exactly one 201 and one 409") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..484d40f --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Additional grade-4 negative and access tests. Requires running Compose stack.""" +import hashlib +import zlib +import sys +import uuid +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, arguments, run + +ROOT = Path(__file__).resolve().parents[1] + +def main(): + args = arguments("Lab5: file validation, permissions, idempotency and isolation") + c = Client(args.base_url, args.ca_file, timeout=35) + alice, bob, eve, chat = c.bootstrap() + file = (ROOT / "tests/fixtures/sample.png").read_bytes() + path = "/api/v1/chats/" + chat["id"] + upload = path + "/attachments?filename=photo.png" + + # 1. Загружаем (теперь ожидаем 202 вместо 201) + result = c.call("POST",upload,202,actor=alice,raw=file, + headers={"Content-Type":"image/png"},schema="Attachment")["json"] + obj = "/api/v1/attachments/" + result["id"] + check(result["sha256"] == hashlib.sha256(file).hexdigest(),"checksum mismatch") + + # Ждем, пока воркер обработает картинку (переведет в статус ready) + result = c.wait_ready(alice, result) + + # Теперь контент доступен + check(c.call("GET",obj+"/content",actor=bob)["bytes"] == file,"original differs") + c.call("GET",obj,404,actor=eve) + c.call("GET",obj+"/content",404,actor=eve) + c.call("GET","/api/v1/attachments/"+str(uuid.uuid4()),404,actor=bob) + + # Негативные тесты (они падают сразу на этапе валидации, до 202) + c.call("POST",upload,415,actor=alice,raw=b"not an image", + headers={"Content-Type":"image/png"}) + c.call("POST",upload,415,actor=alice,raw=file, + headers={"Content-Type":"image/jpeg"}) + c.call("POST",upload,413,actor=alice,raw=b"\0"*10485761, + headers={"Content-Type":"image/png"}) + + huge = bytearray(file) + huge[16:20] = (10000).to_bytes(4, "big") + huge[20:24] = (5000).to_bytes(4, "big") + huge[29:33] = (zlib.crc32(huge[12:29]) & 0xffffffff).to_bytes(4, "big") + c.call("POST",upload,413,actor=alice,raw=bytes(huge), + headers={"Content-Type":"image/png"}) + + c.call("POST",path+"/attachments?filename=..%2Fhidden.png",400,actor=alice, + raw=file,headers={"Content-Type":"image/png"}) + c.call("POST",path+"/attachments?filename=sample.txt",415,actor=alice, + raw=file,headers={"Content-Type":"text/plain"}) + + second = c.call("POST","/api/v1/chats",201,actor=alice, + body={"title":"Other Chat","member_ids":[alice["id"]]},schema="Chat")["json"] + c.call("POST", "/api/v1/chats/"+second["id"]+"/messages",404, + actor=alice,body={"text":"","attachment_ids":[result["id"]]}, + headers={"Idempotency-Key":uuid.uuid4().hex}) + + # В 5-й лабе можно прикреплять файлы в статусе queued/processing/ready + attached = c.message(alice,chat,"",attachments=[result["id"]])["json"] + check(attached["attachment_ids"] == [result["id"]],"message lost attachment") + + key = uuid.uuid4().hex + one = c.message(alice,chat,"Фото",key=key,attachments=[result["id"]])["json"] + check(c.message(alice,chat,"Фото",key=key,attachments=[result["id"]])["json"] == one, + "repeated idempotency response mismatch") + c.call("POST",path+"/messages",409,actor=alice, + body={"text":"Фото", "attachment_ids":[]},headers={"Idempotency-Key":key}) + + print("PASS: private downloads, MIME spoof, corrupt image, oversize, dangerous filename, foreign chat, idempotency") + +if __name__ == "__main__": + run(main) \ No newline at end of file