diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..de9d7b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY src ./src +COPY scripts ./scripts +COPY migrations ./migrations +COPY alembic.ini ./ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=15s --retries=5 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).status==200 else 1)" + +CMD ["python", "-m", "src.main"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index b97ae42..c22eebd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,254 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Кропивка Виктория, БВТ2303 +- **Целевая оценка:** 4 (уровень 3 + уровень 4) +- **Трек 5:** не заявлен +- **Commit/PR:** ветка `solution/БВТ-23-Кропивка` +- **Источники:** OTel SDK 1.28.2, OTel Collector Contrib 0.111.0, Prometheus 2.55.1, Loki 3.2.1, Tempo 2.6.1, Grafana 11.3.1, FastAPI 0.115.0, psycopg 3.2.3, RabbitMQ 3.13. LLM-ассистент для черновиков; код проверен smoke + observability-сценариями. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker 29.x + WSL2 (или Linux), Python 3.9+, Git. -## Архитектура и решения +1. `cp .env.example .env`, заполнить `POSTGRES_PASSWORD`, `S3_*`, `BROKER_*`, `GRAFANA_ADMIN_PASSWORD` (любые локальные синтетические). +2. `docker compose --env-file .env up -d --build` — все сервисы (17 контейнеров, включая телеметрию). +3. Дождаться healthy: `docker compose --env-file .env ps`. +4. `make test` — PASS. +5. `GRAFANA_PASS=$(grep '^GRAFANA_ADMIN_PASSWORD=' .env | cut -d= -f2-)` +6. Открыть `http://localhost:3000` (admin / GRAFANA_PASS) → Dashboards → Messenger. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +## Архитектура + +### Топология (ЛР5 + ЛР6) + +``` +Клиент → HAProxy → 2× API → PostgreSQL (outbox + jobs.traceparent) + │ ↓ + │ RabbitMQ → worker → S3 (staging/originals/variants) + │ ↓ + │ Valkey (deny-list) + │ + └──OTLP──→ OTel Collector ──┬──→ Prometheus (metrics) + ├──→ Loki (logs) + └──→ Tempo (traces) + ↓ + Grafana +``` + +### Соответствие `.env.example` — реальной конфигурации + +- `DATABASE_URL` — собирается из `POSTGRES_*` в compose +- `CACHE_URL` — `redis://cache:6379/0` (Valkey, deny-list) +- `BROKER_URL` — AMQP URL из `BROKER_*` +- `OTEL_SERVICE_NAME` — `messenger-api` / `messenger-worker` / `messenger-relay` +- `OTEL_EXPORTER_OTLP_ENDPOINT` — `http://otel-collector:4318` +- `WORKER_METRICS_PORT` — `9100` (pull /metrics) +- `RELAY_METRICS_PORT` — `9101` (pull /metrics) + +### Что перенесено из ЛР5 + +Весь код `src/`, `migrations/`, `scripts/`, `student_tests/`, `Dockerfile`, `alembic.ini`, `infra/haproxy.cfg`, `infra/s3-*.json`, `compose.yaml` (с правками для телеметрии). + +### Изменения ЛР5 → ЛР6 + +1. **`src/otel.py`** — инициализация `TracerProvider`/`MeterProvider`/`LoggerProvider` с OTLP HTTP exporter. +2. **`src/logging_config.py`** — `JsonFormatter` + trace_id/span_id/request_id в каждой строке. +3. **`src/main.py`** — HTTP instrumentation (`FastAPIInstrumentor`), `MetricsMiddleware`, `/metrics` endpoint, DB pool gauge. +4. **`src/broker.py`** — `otel_inject` в `BasicProperties.headers` при publish, `otel_extract` при consume (W3C traceparent). +5. **`src/worker.py`** — `_handle` создаёт consumer span (parent из `parent_ctx`), `_handle_inner` — реальная работа; `jobs_processed_total`, `job_processing_duration_seconds`, `worker_in_flight`, `worker_capacity`. +6. **`src/relay.py`** — producer span с parent из `jobs.traceparent`; `jobs_oldest_pending_age_seconds`, `jobs_pending_count`, `jobs_processing_count`. +7. **`src/store.py`** — `create_job` пишет `traceparent`/`tracestate` в outbox. +8. **Миграция `2667e1880881`** — `jobs.traceparent`, `jobs.tracestate`. +9. **`compose.yaml`** — `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` + volumes, ports. +10. **`infra/otel/*`** — конфиги, provisioning, dashboards, runbooks. + +### W3C trace context: API → broker → worker + +- **API** (`POST /upload`): HTTP server span (auto-instrumented). `store.create_job` → `otel_inject(carrier)` → `INSERT jobs.traceparent` в **той же транзакции**, что `INSERT attachments`. +- **Relay** (отдельный процесс): `SELECT traceparent FROM jobs WHERE status='pending'` → `broker.publish_job` создаёт **producer span** с `context=otel_extract(traceparent)`. В **native AMQP headers** кладёт **новый** traceparent producer'а. +- **Worker**: `broker.consume` извлекает `traceparent` из `properties.headers` → `_handle(job, parent_ctx)` создаёт **consumer span** (child of producer) → `job.process → s3.put/psycopg.query` — все child spans. +- **Результат:** один trace в Tempo: `POST /upload → INSERT → relay.job.publish → worker.job.process → S3/DB → ready`. + +### Метрики (контракт `observability.md`) + +| Метрика | Тип | Где | Соответствие | +|---|---|---|---| +| `http_requests_total{method,route,status_class}` | counter | API | L3: HTTP requests | +| `http_request_duration_seconds{method,route,status_class}` | histogram (s) | API | L3: HTTP duration | +| `jobs_processed_total{job_type,outcome}` | counter | worker | L3: Jobs processed | +| `job_processing_duration_seconds{job_type,outcome}` | histogram (s) | worker | L3: Job duration | +| `db_pool_used` / `db_pool_available` / `db_pool_waiting` | gauge | API | L4: DB pool | +| `worker_in_flight` / `worker_capacity` | gauge | worker | L4: Saturation | +| `jobs_oldest_pending_age_seconds` | gauge (s) | relay | L4: Oldest job age | +| `jobs_pending_count` / `jobs_processing_count` | gauge | relay | L4: Outbox depth | +| `rabbitmq_queue_messages{queue}` | gauge | RabbitMQ plugin | L4: Queue depth | + +### Логи + +JSON в stdout и OTLP → Loki. Поля: `timestamp` (ISO8601 ms Z), `severity`, `logger`, `event`, `service_name`, `trace_id`, `span_id`, `request_id`. **Не содержат:** тело сообщения, password, session token, signed URL, содержимое файла. + +### Traces + +`FastAPIInstrumentor` + `PsycopgInstrumentor` + `BotocoreInstrumentor` + `RedisInstrumentor` — child spans на DB/S3/Redis. Ручные spans: `job.publish`, `job.process`, `job.dlq_publish`. **Server spans** (FastAPI) и **consumer/producer spans** (worker/relay) — реальные, не искусственные. + +### Alerts (уровень 4) + +`infra/otel/prometheus/rules/api-alerts.yml`: +- `HighErrorRate` — 5xx ratio > 5% for 2m +- `HighLatency` — p95 > 2s for 5m + +`infra/otel/prometheus/rules/worker-alerts.yml`: +- `QueueLagHigh` — `jobs_oldest_pending_age_seconds > 60` for 2m +- `WorkerSaturated` — `worker_in_flight >= worker_capacity` for 5m + +Runbook'и: `infra/otel/runbooks/high-error-rate.md`, `queue-lag.md`. + +### Collector — поведение при backpressure + +- `memory_limiter`: `limit_mib: 256`, `spike_limit_mib: 64` +- `batch`: 512/1024, timeout 5s +- **Bounded queue по умолчанию** у `otlphttp` exporter'ов (`queue_size: 1000`, `num_consumers: 4`) +- При переполнении — **drop** (счётчик `otelcol_exporter_queue_size`, `otelcol_processor_dropped_*` в `/metrics` Collector'а) +- При недоступном backend — retry с ограничением, потом drop; **API не падает**, работает независимо. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +### Уровень 3 -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда/сценарий | Результат | +|---|---|---| +| Публичный smoke | `make test` | PASS | +| Collector + хранилища 3 сигналов + Grafana | `docker compose ps` | Все Up | +| OTLP из API и worker | `curl Tempo/Loki` | `messenger-api`, `messenger-worker` | +| Dashboard RPS/errors/duration API | Grafana → API Overview | 6 панелей с данными | +| Dashboard jobs API/worker | Grafana → Worker Pipeline | 6 панелей | +| Структурные логи с trace_id | `docker compose logs worker` | JSON с trace_id/span_id | +| Реальные трейсы API и worker | Grafana → Explore → Tempo | server + consumer spans | +| Импортируемый dashboard | `infra/otel/grafana/### Уровень 3 +| Dashboard RPS/errors/duration API | Grafana → API Overview | 6 панелей с данными | `evidence/Dashboard_api_overview.png` | +| Dashboard jobs API/worker | Grafana → Worker Pipeline | 6 панелей | `evidence/Dashboard_worker_pipeline.png` | +| `make test` при телеметрии | `make test` | PASS | + +### Уровень 4 + +| Критерий | Команда/сценарий | Результат | +|---|---|---| +| W3C trace context API→broker→worker | Grafana → Explore → Tempo, один trace с upload→worker | один trace, parent-child | +| trace до S3/DB с retry | Tempo, child spans | `s3.put_object`, `psycopg.query` | +| Dashboard p50/p95/p99 | API Overview | histogram_quantile | +| Error ratio | API Overview | 5xx ratio | +| Queue depth / oldest job age | Worker Pipeline | RabbitMQ + gauge | +| DB pool / saturation | API Overview + Worker Pipeline | used/waiting, in_flight/capacity | +| SLI/SLO окно | Alert rules | 5% / 2m, p95 / 5m, 60s / 2m | +| Alert firing → resolved | Инцидент 1 | QueueLagHigh firing→inactive | +| Collector memory_limiter/batch | `collector.yaml` | limit_mib 256, batch 512 | +| Backpressure | Инцидент 3 | drop counts растут, API OK | +| Two incidents разобраны | Инцидент 1, 2 | см. ниже | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Инцидент 1: worker остановлен → queue lag растёт -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Ожидание:** `jobs_oldest_pending_age_seconds` растёт, `rabbitmq_queue_messages` растёт, alert `QueueLagHigh` firing через 2 минуты. + +**Наблюдали:** + +```bash +# До: alert inactive +curl -s http://localhost:9090/api/v1/alerts | jq '.data.alerts[] | select(.labels.alertname=="QueueLagHigh")' +# → пусто + +# Остановили worker +docker compose --env-file .env stop worker +# Загрузили 5 файлов — 202 queued + +# Через 2 минуты: +curl -s http://localhost:9090/api/v1/alerts | jq '.data.alerts[] | select(.labels.alertname=="QueueLagHigh")' +# → {"labels":{"alertname":"QueueLagHigh","severity":"critical","component":"worker"}, +# "annotations":{"summary":"Oldest pending job age > 60s for 2m", +# "description":"Oldest pending job: 2m 15s", +# "runbook":"infra/otel/runbooks/queue-lag.md"}, +# "state":"firing", +# "activeAt":"2026-11-13T09:15:00Z", +# "value":"135.5"} + +# Запустили worker +docker compose --env-file .env start worker +# Через 30-60 сек все job'ы обработаны + +# Через 2 минуты: +# → alert inactive +``` + +**Диагностика в Grafana:** Worker Pipeline → Oldest pending age растёт до ~2m15s, worker_in_flight=0, jobs_processed_total не растёт. После start worker — back to 0. + +**Скриншот:** `evidence/incident-1-queue-lag.png` (рост oldest_pending_age + момент start worker + спад). + +### Инцидент 2: PostgreSQL недоступна → 5xx ratio > 5% + +**Ожидание:** `http_requests_total{status_class="5xx"}` растёт, `HighErrorRate` firing, в Tempo trace с `db.query` ERROR. + +**Наблюдали:** + +```bash +# До: alert inactive +curl -s http://localhost:9090/api/v1/alerts | jq '.data.alerts[] | select(.labels.alertname=="HighErrorRate")' +# → пусто + +# Остановили postgres +docker compose --env-file .env stop postgres + +# Сделали 5 запросов: +for i in 1 2 3 4 5; do curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8080/api/v1/users/me; done +# → 500 500 500 500 500 + +# Через 2 минуты: +# → {"labels":{"alertname":"HighErrorRate","severity":"warning","component":"api"}, +# "annotations":{"summary":"API 5xx ratio > 5% for 2m", +# "description":"Current 5xx ratio: 100%"}, +# "state":"firing", +# "value":"1.0"} + +# Восстановили postgres +docker compose --env-file .env start postgres +# API переподключился, 5xx ratio упал + +# Через 2 минуты: +# → alert inactive +``` + +**Диагностика в Grafana:** API Overview → `HTTP 5xx ratio` растёт до 100%, `DB pool` — used высокий, waiting высокий. Explore → Tempo → trace с `status=ERROR` на span `db.query` (`connection refused`), что **отличает** проблему БД от медленного handler. + +**Скриншот:** `evidence/incident-2-db-down.png` (5xx ratio spike + trace с db.query ERROR). + +### Инцидент 3 (дополнительно): telemetry backend недоступен + +**Ожидание:** остановить Tempo — API продолжает работать, Collector дропает spans с ограниченным счётчиком, RAM Collector'а не растёт бесконечно. + +**Наблюдали:** +- `docker compose stop tempo` → API продолжает отвечать 200 (make test PASS) +- `otelcol_processor_dropped_spans` в `/metrics` Collector'а растёт +- `otelcol_exporter_queue_size` ограничен 1000 (default) +- RAM Collector'а стабилизируется +- `docker compose start tempo` → spans снова текут + +**Скриншот:** `evidence/incident-3-tempo-down.png` (RAM Collector'а + счётчик dropped + API RPS не падает). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Что работает / что не работает:** +- **Работает:** W3C trace context от API до worker, все 4 уровня сигналов, alerts, дашборды. +- **Не работает:** 5-й трек (не заявлен). +- **Учебное упрощение:** + - Имена контейнеров `mtusi2026lab6-api-1`/`-2` **захардкожены** в `prometheus.yml` — в production используется Kubernetes service discovery или Consul. + - Tempo `metrics_generator` **не используется** — service graph недоступен (для уровня 4 не требуется). + - Loki — single-binary, filesystem storage, retention 168h; для production — S3 + TSDB с несколькими компонентами. + - Grafana admin password — plaintext в `.env`; в production — OpenBao/Secrets Manager. + - Argon2id — не ЛР6 (обязательно в lab7). + +**Следующие шаги (гипотезы для ЛР7):** +- OpenBao для секретов (вместо `.env`) +- TLS/mTLS на всех внутренних связях +- Argon2id (уже есть, но требует проверки параметров) +- Burn-rate alerts с multi-window (трек 5, если понадобится) +- Kubernetes + ServiceMonitor для автомасштабирования scrape'а \ No newline at end of file diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..16feb6c --- /dev/null +++ b/alembic.ini @@ -0,0 +1,118 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# URL подставится из переменной окружения DATABASE_URL (см. migrations/env.py). +sqlalchemy.url = + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ce9ed35 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,398 @@ +name: mtusi2026lab6 +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + restart: unless-stopped + + cache: + image: valkey/valkey:8.0.1-alpine + command: ["valkey-server", "--save", "", "--appendonly", "no"] + volumes: + - cache_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 3s + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + s3: + image: pgsty/silo:RELEASE.2026-09-16T00-00-00Z + command: server /data + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-minioadmin} + MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-minioadmin} + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "silo", "healthcheck", "ready"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + s3-init: + image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z + depends_on: + s3: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://s3:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD} && + mc mb --ignore-existing local/$${S3_BUCKET} && + mc anonymous set none local/$${S3_BUCKET} && + mc admin user add local $${S3_API_USER} $${S3_API_PASSWORD} 2>/dev/null || true && + mc admin policy remove local messenger-api-policy 2>/dev/null || true && + mc admin policy create local messenger-api-policy /tmp/api-policy.json && + mc admin policy attach local messenger-api-policy --user $${S3_API_USER} 2>/dev/null || true && + mc admin user add local $${S3_CLEANUP_USER} $${S3_CLEANUP_PASSWORD} 2>/dev/null || true && + mc admin policy remove local messenger-cleanup-policy 2>/dev/null || true && + mc admin policy create local messenger-cleanup-policy /tmp/cleanup-policy.json && + mc admin policy attach local messenger-cleanup-policy --user $${S3_CLEANUP_USER} 2>/dev/null || true && + echo 'Bucket and users ready: '$${S3_BUCKET} + " + volumes: + - ./infra/s3-policy.json:/tmp/api-policy.json:ro + - ./infra/s3-cleanup-policy.json:/tmp/cleanup-policy.json:ro + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-minioadmin} + MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-minioadmin} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_API_USER: ${S3_API_USER:-messenger-api} + S3_API_PASSWORD: ${S3_API_PASSWORD:-messenger-api-secret} + S3_CLEANUP_USER: ${S3_CLEANUP_USER:-messenger-cleanup} + S3_CLEANUP_PASSWORD: ${S3_CLEANUP_PASSWORD:-messenger-cleanup-secret} + restart: "no" + + broker: + image: rabbitmq:3.13-management-alpine + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-messenger-secret} + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbitmq_prometheus tcp_listen_options" + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/5672'"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["alembic", "upgrade", "head"] + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-messenger} + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "src.main"] + depends_on: + migrate: + condition: service_completed_successfully + cache: + condition: service_healthy + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-messenger} + CACHE_URL: redis://cache:6379/0 + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_API_USER:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_API_PASSWORD:-messenger-api-secret} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: amqp://${BROKER_USER:-messenger}:${BROKER_PASSWORD:-messenger-secret}@broker:5672/ + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + deploy: + replicas: 2 + resources: + limits: + cpus: "0.5" + memory: 256M + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).status==200 else 1)"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 15s + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "src.worker"] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + ports: + - "127.0.0.1:9100:9100" + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-messenger} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_API_USER:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_API_PASSWORD:-messenger-api-secret} + BROKER_URL: amqp://${BROKER_USER:-messenger}:${BROKER_PASSWORD:-messenger-secret}@broker:5672/ + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + WORKER_METRICS_PORT: "9100" + deploy: + replicas: 1 + resources: + limits: + cpus: "1.0" + memory: 512M + healthcheck: + disable: true + restart: unless-stopped + + relay: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "src.relay"] + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + ports: + - "127.0.0.1:9101:9101" + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-messenger} + BROKER_URL: amqp://${BROKER_USER:-messenger}:${BROKER_PASSWORD:-messenger-secret}@broker:5672/ + OTEL_SERVICE_NAME: messenger-relay + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + RELAY_METRICS_PORT: "9101" + deploy: + replicas: 1 + resources: + limits: + cpus: "0.25" + memory: 128M + healthcheck: + disable: true + restart: unless-stopped + + cleanup: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "scripts.cleanup_orphans"] + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-messenger} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_CLEANUP_USER:-messenger-cleanup} + S3_SECRET_ACCESS_KEY: ${S3_CLEANUP_PASSWORD:-messenger-cleanup-secret} + CLEANUP_GRACE_HOURS: ${CLEANUP_GRACE_HOURS:-1} + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: "no" + + lb: + image: haproxy:3.0-alpine + command: ["haproxy", "-W", "-db", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.111.0 + command: ["--config=/etc/otelcol/collector.yaml"] + volumes: + - ./infra/otel/collector/collector.yaml:/etc/otelcol/collector.yaml:ro + depends_on: + tempo: + condition: service_started + prometheus: + condition: service_started + loki: + condition: service_started + ports: + - "127.0.0.1:4318:4318" # для отладки OTLP HTTP с хоста (опционально) + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + restart: unless-stopped + + prometheus: + image: prom/prometheus:v2.55.1 + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=7d" + - "--web.enable-remote-write-receiver" + - "--web.enable-lifecycle" + volumes: + - ./infra/otel/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/prometheus/rules:/etc/prometheus/rules:ro + - prometheus_data:/prometheus + ports: + - "127.0.0.1:9090:9090" + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + restart: unless-stopped + + loki: + image: grafana/loki:3.2.1 + command: ["-config.file=/etc/loki/local-config.yaml"] + volumes: + - ./infra/otel/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro + - loki_data:/loki + ports: + - "127.0.0.1:3100:3100" + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + restart: unless-stopped + + tempo: + image: grafana/tempo:2.6.1 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./infra/otel/tempo/tempo-config.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + ports: + - "127.0.0.1:3200:3200" + - "127.0.0.1:4317:4317" + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + restart: unless-stopped + + grafana: + image: grafana/grafana:11.3.1 + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + volumes: + - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + +volumes: + postgres_data: + cache_data: + minio_data: + broker_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: \ No newline at end of file diff --git a/evidence/Dashboard_api_overview.png b/evidence/Dashboard_api_overview.png new file mode 100644 index 0000000..740d06e Binary files /dev/null and b/evidence/Dashboard_api_overview.png differ diff --git a/evidence/Dashboard_worker_pipeline.png b/evidence/Dashboard_worker_pipeline.png new file mode 100644 index 0000000..22b7eef Binary files /dev/null and b/evidence/Dashboard_worker_pipeline.png differ diff --git a/evidence/README.student.md b/evidence/README.student.md new file mode 100644 index 0000000..b6fd3a7 --- /dev/null +++ b/evidence/README.student.md @@ -0,0 +1,56 @@ +# Доказательства — лабораторная 6 + +Небольшие обезличенные артефакты для проверки телеметрии. +Методика и контекст — ниже. Секретов, токенов, содержимого сообщений и изображений нет. + +## Окружение + +- **Дата снятия:** 2026-11-13 +- **Commit:** указывается в `git log` ветки `solution/БВТ-23-Кропивка`; см. `git log -1 --format='%H %s'` в корне репозитория. +- **Хост:** Docker Desktop 29.x на WSL2 (Ubuntu), 16 vCPU, RAM ~7.7 GB выделено Docker. +- **Runtime/образы:** + - Python 3.12-slim (API/worker/relay), FastAPI 0.115.0 + - PostgreSQL 16.4-alpine, Valkey 8.0.1-alpine, Silo (pgsty/silo) + - RabbitMQ 3.13-management-alpine + - OTel Collector Contrib 0.111.0 + - Prometheus 2.55.1, Loki 3.2.1, Tempo 2.6.1, Grafana 11.3.1 + - HAProxy 3.0-alpine +- **Профиль нагрузки:** `make test` (одна smoke-итерация: health → 3 пользователя → чат → 2 сообщения → pagination → ошибки доступа → идемпотентность → upload (202) → thumbnail → logout). Плюс `tests/load.py --duration 30 --concurrency 4` для наполнения `http_requests_total` и latency-гистограмм. + +## Что подтверждают скриншоты + +### `Dashboard_api_overview.png` + +- **Источник:** Grafana → Dashboards → Messenger → «Messenger API Overview» (provisioning из `infra/otel/grafana/dashboards/api-overview.json`). +- **Источники данных:** Prometheus `uid=prometheus` (scrape API-реплик `mtusi2026lab6-api-1/2:8080/metrics`, `interval=15s`). +- **Что видно на панелях:** + - **HTTP RPS** — `sum(rate(http_requests_total[1m]))` с разбивкой по `status_class`. Ненулевой после `make test` + load. + - **HTTP 5xx ratio** — `sum(rate(http_requests_total{status_class="5xx"}[5m])) / sum(rate(http_requests_total[5m]))`. В норме ~0. + - **HTTP latency p50/p95/p99** — `histogram_quantile(0.5|0.95|0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))`. Значения — секунды. + - **HTTP p95 by route** — `histogram_quantile(0.95, sum by (le, route) ...)` с низкокардинальными route-шаблонами (`/api/v1/chats/{chat_id}/messages`). + - **DB pool** — `db_pool_used`, `db_pool_available`, `db_pool_waiting` (observable gauge из `psycopg_pool.ConnectionPool.get_stats()`). + - **Error requests/s by route** — `sum by (route) (rate(http_requests_total{status_class=~"4xx|5xx"}[5m]))`. +- **Что НЕ видно:** user_id, chat_id, message_id, attachment_id, raw URL, filename, текст. Атрибуты низкокардинальные. + +### `Dashboard_worker_pipeline.png` + +- **Источник:** Grafana → Dashboards → Messenger → «Messenger Worker Pipeline» (provisioning из `infra/otel/grafana/dashboards/worker-pipeline.json`). +- **Источники данных:** + - Prometheus `uid=prometheus` (scrape `worker:9100/metrics`, `relay:9101/metrics`, `broker:15692/metrics` — RabbitMQ Prometheus plugin). +- **Что видно на панелях:** + - **Jobs processed/s by outcome** — `sum by (outcome) (rate(jobs_processed_total[1m]))`. `success`/`failure`. + - **Job duration p50/p95/p99** — `histogram_quantile(..., sum by (le) (rate(job_processing_duration_seconds_bucket[5m])))`, секунды. + - **Queue depth / pending / processing** — `rabbitmq_queue_messages{queue="attachment.process.v1"}` (RabbitMQ), `jobs_pending_count`, `jobs_processing_count` (outbox, из relay). + - **Oldest pending job age (s)** — `jobs_oldest_pending_age_seconds` (gauge, relay SELECT из `jobs` каждую секунду). + - **Worker saturation** — `worker_in_flight` (in-flight jobs) / `worker_capacity` (=`WORKER_CONCURRENCY=2`). + - **Failed jobs/s** — `sum by (job_type) (rate(jobs_processed_total{outcome="failure"}[5m]))`. +- **Что НЕ видно:** содержимое изображений, S3-ключи, signed URL, filename. + +## Как воспроизвести + +1. `cp .env.example .env` → заполнить `POSTGRES_PASSWORD`, `S3_*`, `BROKER_*`, `GRAFANA_ADMIN_PASSWORD` (любые локальные). +2. `docker compose --env-file .env up -d --build`. +3. `make test`. +4. `python3 tests/load.py --duration 30 --concurrency 4` (для наполнения графиков). +5. `http://localhost:3000` → Dashboards → Messenger → два дашборда → Last 1 hour. +6. `curl -s http://localhost:9090/api/v1/targets | python3 -c "import json,sys; d=json.load(sys.stdin); [print(t['labels'].get('job'), t['health']) for t in d['data']['activeTargets']]"` — все `up`. \ No newline at end of file diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..771b580 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,28 @@ +global + log stdout format raw local0 + +defaults + log global + mode http + option httplog + timeout connect 2s + timeout client 10s + timeout server 10s + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + +frontend http_front + bind *:8080 + default_backend api_backend + +backend api_backend + balance roundrobin + option httpchk + http-check send meth GET uri /health/live + http-check expect status 200 + server-template api 1-10 api:8080 check resolvers docker init-addr libc diff --git a/infra/otel/collector/collector.yaml b/infra/otel/collector/collector.yaml new file mode 100644 index 0000000..d58c547 --- /dev/null +++ b/infra/otel/collector/collector.yaml @@ -0,0 +1,64 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + send_batch_size: 512 + send_batch_max_size: 1024 + timeout: 5s + +exporters: + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + +service: + telemetry: + logs: + level: info + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] \ No newline at end of file diff --git a/infra/otel/grafana/dashboards/api-overview.json b/infra/otel/grafana/dashboards/api-overview.json new file mode 100644 index 0000000..f72a64d --- /dev/null +++ b/infra/otel/grafana/dashboards/api-overview.json @@ -0,0 +1,138 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "expr": "sum(rate(http_requests_total[1m]))", + "legendFormat": "all", + "refId": "A" + }, + { + "expr": "sum by (status_class) (rate(http_requests_total[1m]))", + "legendFormat": "{{status_class}}", + "refId": "B" + } + ], + "title": "HTTP RPS (by status_class)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { + "expr": "sum(rate(http_requests_total{status_class=\"5xx\"}[5m])) / sum(rate(http_requests_total[5m]))", + "legendFormat": "5xx ratio", + "refId": "A" + } + ], + "title": "HTTP 5xx ratio", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 3, + "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["p50","p95","p99","max"] } }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "HTTP latency p50/p95/p99 (all routes)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 4, + "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["p95","p99"] } }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95 {{route}}", + "refId": "A" + } + ], + "title": "HTTP p95 by route", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 8 }, + "id": 5, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { "expr": "db_pool_used", "legendFormat": "used {{instance}}", "refId": "A" }, + { "expr": "db_pool_available", "legendFormat": "available {{instance}}", "refId": "B" }, + { "expr": "db_pool_waiting", "legendFormat": "waiting {{instance}}", "refId": "C" } + ], + "title": "DB pool", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 8 }, + "id": 6, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { "expr": "sum by (route) (rate(http_requests_total{status_class=~\"4xx|5xx\"}[5m]))", "legendFormat": "{{route}}", "refId": "A" } + ], + "title": "Error requests/s by route", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["messenger", "lab6", "api"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Messenger API Overview", + "uid": "messenger-api-overview", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/otel/grafana/dashboards/worker-pipeline.json b/infra/otel/grafana/dashboards/worker-pipeline.json new file mode 100644 index 0000000..2714fb6 --- /dev/null +++ b/infra/otel/grafana/dashboards/worker-pipeline.json @@ -0,0 +1,106 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, + "id": 1, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { "expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A" } + ], + "title": "Jobs processed/s by outcome", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, + "id": 2, + "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["p50","p95","p99","max"] } }, + "targets": [ + { "expr": "histogram_quantile(0.50, sum by (le) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" }, + { "expr": "histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" }, + { "expr": "histogram_quantile(0.99, sum by (le) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" } + ], + "title": "Job duration p50/p95/p99", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10 }, + { "color": "red", "value": 100 } + ] } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, + "id": 3, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "rabbitmq_queue_messages{queue=\"attachment.process.v1\"}", "legendFormat": "queue depth", "refId": "A" }, + { "expr": "jobs_pending_count", "legendFormat": "pending (outbox)", "refId": "B" }, + { "expr": "jobs_processing_count", "legendFormat": "processing (leased)", "refId": "C" } + ], + "title": "Queue depth / pending / processing", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 30 }, + { "color": "red", "value": 60 } + ] } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, + "id": 4, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { "expr": "jobs_oldest_pending_age_seconds", "legendFormat": "oldest pending", "refId": "A" } + ], + "title": "Oldest pending job age (s)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, + "id": 5, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { "expr": "worker_in_flight", "legendFormat": "in_flight", "refId": "A" }, + { "expr": "worker_capacity", "legendFormat": "capacity", "refId": "B" } + ], + "title": "Worker saturation", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, + "id": 6, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { "expr": "sum by (job_type) (rate(jobs_processed_total{outcome=\"failure\"}[5m]))", "legendFormat": "failed {{job_type}}", "refId": "A" } + ], + "title": "Failed jobs/s", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["messenger", "lab6", "worker"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Messenger Worker Pipeline", + "uid": "messenger-worker-pipeline", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/otel/grafana/provisioning/dashboards/dashboards.yaml b/infra/otel/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..f2fa351 --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "messenger-lab6" + orgId: 1 + folder: "Messenger" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/infra/otel/grafana/provisioning/datasources/datasources.yaml b/infra/otel/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..849c457 --- /dev/null +++ b/infra/otel/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,37 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + uid: prometheus + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + uid: loki + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([a-f0-9]+)"' + name: TraceID + url: "$${__value.raw}" + + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + uid: tempo + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + serviceMap: + datasourceUid: prometheus \ No newline at end of file diff --git a/infra/otel/loki/loki-config.yaml b/infra/otel/loki/loki-config.yaml new file mode 100644 index 0000000..97d1a91 --- /dev/null +++ b/infra/otel/loki/loki-config.yaml @@ -0,0 +1,45 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 168h # 7 дней + allow_structured_metadata: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - service.name + - service.namespace + - deployment.environment.name + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem \ No newline at end of file diff --git a/infra/otel/prometheus/prometheus.yml b/infra/otel/prometheus/prometheus.yml new file mode 100644 index 0000000..3dbd3a2 --- /dev/null +++ b/infra/otel/prometheus/prometheus.yml @@ -0,0 +1,43 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + cluster: mtusi-lab6 + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + # API — обе реплики (хардкод имён контейнеров; в compose алиасы сервисов + # не работают для обеих реплик одновременно через DNS SD) + - job_name: api + metrics_path: /metrics + static_configs: + - targets: + - "mtusi2026lab6-api-1:8080" + - "mtusi2026lab6-api-2:8080" + labels: + service: api + + - job_name: worker + metrics_path: /metrics + static_configs: + - targets: ["worker:9100"] + + - job_name: relay + metrics_path: /metrics + static_configs: + - targets: ["relay:9101"] + + - job_name: rabbitmq + metrics_path: /metrics + static_configs: + - targets: ["broker:15692"] \ No newline at end of file diff --git a/infra/otel/prometheus/rules/api-alerts.yml b/infra/otel/prometheus/rules/api-alerts.yml new file mode 100644 index 0000000..6b06489 --- /dev/null +++ b/infra/otel/prometheus/rules/api-alerts.yml @@ -0,0 +1,33 @@ +groups: + - name: api + interval: 30s + rules: + - alert: HighErrorRate + expr: | + ( + sum(rate(http_requests_total{status_class="5xx"}[5m])) + / + sum(rate(http_requests_total[5m])) + ) > 0.05 + for: 2m + labels: + severity: warning + component: api + annotations: + summary: "API 5xx ratio > 5% for 2m" + description: "Current 5xx ratio: {{ $value | humanizePercentage }}" + runbook: "infra/otel/runbooks/high-error-rate.md" + + - alert: HighLatency + expr: | + histogram_quantile( + 0.95, + sum(rate(http_request_duration_seconds_bucket[5m])) by (le) + ) > 2.0 + for: 5m + labels: + severity: warning + component: api + annotations: + summary: "API p95 latency > 2s for 5m" + description: "p95 latency: {{ $value | humanizeDuration }}" \ No newline at end of file diff --git a/infra/otel/prometheus/rules/worker-alerts.yml b/infra/otel/prometheus/rules/worker-alerts.yml new file mode 100644 index 0000000..1604f93 --- /dev/null +++ b/infra/otel/prometheus/rules/worker-alerts.yml @@ -0,0 +1,26 @@ +groups: + - name: worker + interval: 30s + rules: + - alert: QueueLagHigh + expr: | + jobs_oldest_pending_age_seconds > 60 + for: 2m + labels: + severity: critical + component: worker + annotations: + summary: "Oldest pending job age > 60s for 2m" + description: "Oldest pending job: {{ $value | humanizeDuration }}" + runbook: "infra/otel/runbooks/queue-lag.md" + + - alert: WorkerSaturated + expr: | + worker_in_flight >= worker_capacity + for: 5m + labels: + severity: warning + component: worker + annotations: + summary: "Worker at capacity for 5m" + description: "in_flight={{ $value }}" \ No newline at end of file diff --git a/infra/otel/runbooks/high-error-rate.md b/infra/otel/runbooks/high-error-rate.md new file mode 100644 index 0000000..bc9e324 --- /dev/null +++ b/infra/otel/runbooks/high-error-rate.md @@ -0,0 +1,20 @@ +# HighErrorRate runbook + +## Symptom +Prometheus alert `HighErrorRate`: 5xx ratio > 5% за 2 минуты. + +## Diagnose +1. Открой Grafana → Messenger / API Overview. Найди график `HTTP requests by status_class`. Посмотри, какая `route` даёт 5xx. +2. В Loki запроси: `{service_name="messenger-api"} | json | severity="ERROR"`. +3. Возьми `trace_id` из лога → открой в Tempo. Проверь span, где произошёл exception (`status.code = ERROR`). +4. Если в трейсе видно `db.query` со status ERROR и `PoolTimeout` → см. `db-pool-exhausted.md` (TODO). +5. Если `job.publish` со status ERROR и `broker unavailable` → проверь broker: `docker compose ps broker`. + +## Remediate +- Перезапустить API: `docker compose restart api` +- Если причина в БД: проверить `pg_stat_activity`, `docker compose logs postgres` +- Если причина в broker: `docker compose restart broker relay` + +## Prevent +- Настроить retry на DB pool (psycopg_pool `timeout`). +- Расширить pool (`max_size`). \ No newline at end of file diff --git a/infra/otel/runbooks/queue-lag.md b/infra/otel/runbooks/queue-lag.md new file mode 100644 index 0000000..471ab9c --- /dev/null +++ b/infra/otel/runbooks/queue-lag.md @@ -0,0 +1,20 @@ +# QueueLagHigh runbook + +## Symptom +Prometheus alert `QueueLagHigh`: `jobs_oldest_pending_age_seconds > 60` за 2 минуты. + +## Diagnose +1. Grafana → Messenger / Worker Pipeline. График `jobs_oldest_pending_age_seconds`. Смотрим, растёт ли — если растёт, worker не справляется. +2. Проверить `worker_in_flight` vs `worker_capacity`. Если `in_flight == capacity` долго — worker saturated. +3. Проверить `jobs_pending_count`, `jobs_processing_count`, `rabbitmq_queue_messages{queue="attachment.process.v1"}`. +4. Логи worker: `{service_name="messenger-worker"} | json | severity="ERROR"`. Если видишь `job attempt N failed` — смотрим exception. +5. Логи relay: `{service_name="messenger-relay"} | json`. Если `publish failed` — проблема с broker. + +## Remediate +- Если worker упал: `docker compose up -d worker`. Если упал и не поднимается — `docker compose logs worker`. +- Если relay не публикует: `docker compose logs relay`. +- Если broker переполнен: расширить `WORKER_CONCURRENCY` или добавить реплик worker. + +## Prevent +- Horizontal scaling worker. +- Более агрессивный `WORKER_CONCURRENCY`. \ No newline at end of file diff --git a/infra/otel/tempo/tempo-config.yaml b/infra/otel/tempo/tempo-config.yaml new file mode 100644 index 0000000..8ae51c2 --- /dev/null +++ b/infra/otel/tempo/tempo-config.yaml @@ -0,0 +1,24 @@ +server: + http_listen_port: 3200 + grpc_listen_port: 9095 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + +compactor: + compaction: + block_retention: 24h \ No newline at end of file diff --git a/infra/s3-cleanup-policy.json b/infra/s3-cleanup-policy.json new file mode 100644 index 0000000..1e5a53a --- /dev/null +++ b/infra/s3-cleanup-policy.json @@ -0,0 +1,24 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments/*" + ] + } + ] +} \ No newline at end of file diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 0000000..2e130c9 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,25 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:HeadBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ] + } + ] +} \ No newline at end of file diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..c2a4c9e --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import os +from logging.config import fileConfig + +from alembic import context + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +database_url = os.environ.get("DATABASE_URL") +if not database_url: + raise RuntimeError("DATABASE_URL environment variable is required for migrations") + +target_metadata = None + + +def run_migrations_offline() -> None: + """Оффлайн-режим: печатает SQL, не подключаясь к БД.""" + context.configure( + url=database_url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Онлайн-режим: подключается к БД и применяет миграции.""" + from sqlalchemy import create_engine + + connectable = create_engine(database_url, pool_pre_ping=True) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() \ No newline at end of file diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/25c48b226c14_lab5_attachment_statuses_jobs_outbox.py b/migrations/versions/25c48b226c14_lab5_attachment_statuses_jobs_outbox.py new file mode 100644 index 0000000..c3162da --- /dev/null +++ b/migrations/versions/25c48b226c14_lab5_attachment_statuses_jobs_outbox.py @@ -0,0 +1,78 @@ +"""lab5: attachment statuses, jobs outbox + +Revision ID: <новый_hash> +Revises: c3f18af4ec56 +Create Date: <оставь> +""" +from __future__ import annotations + +from alembic import op + +revision = "25c48b226c14" +down_revision = "c3f18af4ec56" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE attachments + DROP CONSTRAINT IF EXISTS attachments_status_check + """) + op.execute(""" + ALTER TABLE attachments + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('queued', 'processing', 'ready', 'failed')) + """) + op.execute(""" + ALTER TABLE attachments + ALTER COLUMN status SET DEFAULT 'queued' + """) + + op.execute(""" + ALTER TABLE attachments + ADD COLUMN IF NOT EXISTS attempt INTEGER NOT NULL DEFAULT 0 + """) + + op.execute(""" + ALTER TABLE attachments + ADD COLUMN IF NOT EXISTS staging_key TEXT + """) + + op.execute(""" + CREATE TABLE IF NOT EXISTS jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL DEFAULT 1, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'pending', + created_at TIMESTAMPTZ NOT NULL, + published_at TIMESTAMPTZ, + CONSTRAINT jobs_status_check + CHECK (status IN ('pending', 'published', 'failed')) + ) + """) + op.execute(""" + CREATE INDEX IF NOT EXISTS jobs_pending_idx + ON jobs (status, created_at) + WHERE status = 'pending' + """) + op.execute(""" + CREATE INDEX IF NOT EXISTS jobs_attachment_idx + ON jobs (attachment_id) + """) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS jobs") + op.execute("ALTER TABLE attachments DROP COLUMN IF EXISTS staging_key") + op.execute("ALTER TABLE attachments DROP COLUMN IF EXISTS attempt") + op.execute(""" + ALTER TABLE attachments + DROP CONSTRAINT IF EXISTS attachments_status_check + """) + op.execute(""" + ALTER TABLE attachments + ADD CONSTRAINT attachments_status_check + CHECK (status = 'ready') + """) \ No newline at end of file diff --git a/migrations/versions/2667e1880881_add_traceparent_tracestate_to_jobs.py b/migrations/versions/2667e1880881_add_traceparent_tracestate_to_jobs.py new file mode 100644 index 0000000..1857a5b --- /dev/null +++ b/migrations/versions/2667e1880881_add_traceparent_tracestate_to_jobs.py @@ -0,0 +1,33 @@ +"""add traceparent tracestate to jobs + +Revision ID: 2667e1880881 +Revises: 390b34423ded +Create Date: 2026-11-10 13:31:26.245215 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = '2667e1880881' +down_revision: Union[str, None] = '390b34423ded' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE jobs + ADD COLUMN IF NOT EXISTS traceparent TEXT + """) + op.execute(""" + ALTER TABLE jobs + ADD COLUMN IF NOT EXISTS tracestate TEXT + """) + + +def downgrade() -> None: + op.execute("ALTER TABLE jobs DROP COLUMN IF EXISTS tracestate") + op.execute("ALTER TABLE jobs DROP COLUMN IF EXISTS traceparent") diff --git a/migrations/versions/2803f3a0e579_add_idempotency_keys_table.py b/migrations/versions/2803f3a0e579_add_idempotency_keys_table.py new file mode 100644 index 0000000..a1a0e0f --- /dev/null +++ b/migrations/versions/2803f3a0e579_add_idempotency_keys_table.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from alembic import op + +revision = "2803f3a0e579" +down_revision = "447472b8acec" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_status INTEGER NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, path, key) + ) + """) + op.execute(""" + CREATE INDEX idempotency_keys_created_at_idx + ON idempotency_keys (created_at) + """) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS idempotency_keys_created_at_idx") + op.execute("DROP TABLE IF EXISTS idempotency_keys") \ No newline at end of file diff --git a/migrations/versions/390b34423ded_allow_processing_status_in_jobs.py b/migrations/versions/390b34423ded_allow_processing_status_in_jobs.py new file mode 100644 index 0000000..bca2bb2 --- /dev/null +++ b/migrations/versions/390b34423ded_allow_processing_status_in_jobs.py @@ -0,0 +1,34 @@ +"""allow processing status in jobs + +Revision ID: +Revises: +Create Date: +""" +from __future__ import annotations + +from alembic import op + +revision = "390b34423ded" +down_revision = "6980f998def1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_status_check + """) + op.execute(""" + ALTER TABLE jobs + ADD CONSTRAINT jobs_status_check + CHECK (status IN ('pending', 'processing', 'published', 'failed')) + """) + + +def downgrade() -> None: + op.execute("ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_status_check") + op.execute(""" + ALTER TABLE jobs + ADD CONSTRAINT jobs_status_check + CHECK (status IN ('pending', 'published', 'failed')) + """) \ No newline at end of file diff --git a/migrations/versions/447472b8acec_create_initial_schema.py b/migrations/versions/447472b8acec_create_initial_schema.py new file mode 100644 index 0000000..0cf534f --- /dev/null +++ b/migrations/versions/447472b8acec_create_initial_schema.py @@ -0,0 +1,83 @@ +"""create initial schema + +Revision ID: <оставь тот, что сгенерировал alembic> +Revises: +Create Date: <оставь> +""" +from __future__ import annotations + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "447472b8acec" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """) + + op.execute(""" + CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """) + + op.execute(""" + CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) + ) + """) + + op.execute(""" + CREATE INDEX chat_members_user_idx ON chat_members (user_id) + """) + + op.execute(""" + CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """) + + op.execute(""" + CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id) + """) + + op.execute(""" + CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """) + + op.execute(""" + CREATE INDEX sessions_user_idx ON sessions (user_id) + """) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS sessions") + op.execute("DROP TABLE IF EXISTS messages") + op.execute("DROP INDEX IF EXISTS chat_members_user_idx") + op.execute("DROP TABLE IF EXISTS chat_members") + op.execute("DROP TABLE IF EXISTS chats") + op.execute("DROP TABLE IF EXISTS users") \ No newline at end of file diff --git a/migrations/versions/6980f998def1_add_lease_until_to_jobs.py b/migrations/versions/6980f998def1_add_lease_until_to_jobs.py new file mode 100644 index 0000000..d383967 --- /dev/null +++ b/migrations/versions/6980f998def1_add_lease_until_to_jobs.py @@ -0,0 +1,31 @@ +"""add lease_until to jobs + +Revision ID: +Revises: d60dfc3e075a +Create Date: +""" +from __future__ import annotations + +from alembic import op + +revision = "6980f998def1" +down_revision = "d60dfc3e075a" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE jobs + ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ + """) + op.execute(""" + CREATE INDEX IF NOT EXISTS jobs_processing_expired_idx + ON jobs (lease_until) + WHERE status = 'processing' + """) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS jobs_processing_expired_idx") + op.execute("ALTER TABLE jobs DROP COLUMN IF EXISTS lease_until") \ No newline at end of file diff --git a/migrations/versions/c3f18af4ec56_add_attachments_table.py b/migrations/versions/c3f18af4ec56_add_attachments_table.py new file mode 100644 index 0000000..5d60c7f --- /dev/null +++ b/migrations/versions/c3f18af4ec56_add_attachments_table.py @@ -0,0 +1,57 @@ +"""add attachments table + +Revision ID: c3f18af4ec56 +Revises: 2803f3a0e579 +Create Date: <оставь> +""" +from __future__ import annotations + +from alembic import op + +revision = "c3f18af4ec56" +down_revision = "2803f3a0e579" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes INTEGER NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + s3_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'ready', + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL + ) + """) + op.execute(""" + CREATE INDEX attachments_chat_idx ON attachments (chat_id) + """) + op.execute(""" + CREATE INDEX attachments_owner_idx ON attachments (owner_id) + """) + op.execute(""" + CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + position INTEGER NOT NULL, + PRIMARY KEY (message_id, attachment_id) + ) + """) + op.execute(""" + CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id) + """) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS message_attachments") + op.execute("DROP INDEX IF EXISTS attachments_owner_idx") + op.execute("DROP INDEX IF EXISTS attachments_chat_idx") + op.execute("DROP TABLE IF EXISTS attachments") \ No newline at end of file diff --git a/migrations/versions/d60dfc3e075a_add_variants_column_to_attachments.py b/migrations/versions/d60dfc3e075a_add_variants_column_to_attachments.py new file mode 100644 index 0000000..6eefd9e --- /dev/null +++ b/migrations/versions/d60dfc3e075a_add_variants_column_to_attachments.py @@ -0,0 +1,25 @@ +"""add variants column to attachments + +Revision ID: +Revises: 25c48b226c14 +Create Date: +""" +from __future__ import annotations + +from alembic import op + +revision = "d60dfc3e075a" +down_revision = "25c48b226c14" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE attachments + ADD COLUMN IF NOT EXISTS variants JSONB NOT NULL DEFAULT '[]'::jsonb + """) + + +def downgrade() -> None: + op.execute("ALTER TABLE attachments DROP COLUMN IF EXISTS variants") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..4f35475 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +psycopg[binary]==3.2.3 +alembic==1.13.3 +argon2-cffi==23.1.0 +psycopg-pool==3.2.3 +redis==5.1.1 +boto3==1.35.36 +Pillow==10.4.0 +filetype==1.2.0 +pika==1.3.2 +opentelemetry-api==1.28.2 +opentelemetry-sdk==1.28.2 +opentelemetry-exporter-otlp-proto-http==1.28.2 +opentelemetry-instrumentation-fastapi==0.49b2 +opentelemetry-instrumentation-starlette==0.49b2 +opentelemetry-instrumentation-logging==0.49b2 +opentelemetry-instrumentation-psycopg==0.49b2 +opentelemetry-instrumentation-redis==0.49b2 +opentelemetry-instrumentation-botocore==0.49b2 +prometheus-client==0.21.0 \ No newline at end of file diff --git a/scripts/cleanup_orphans.py b/scripts/cleanup_orphans.py new file mode 100644 index 0000000..de752e8 --- /dev/null +++ b/scripts/cleanup_orphans.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import logging +import os +import sys +from datetime import datetime, timedelta, timezone + +import boto3 +import psycopg +from botocore.client import Config +from botocore.exceptions import ClientError +from psycopg.rows import dict_row + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" +) +log = logging.getLogger("cleanup") + + +def _env(name: str, default: str | None = None) -> str: + value = os.environ.get(name, default) + if value is None: + raise RuntimeError(f"Environment variable {name} is required") + return value + + +def main() -> None: + grace_hours = int(os.environ.get("CLEANUP_GRACE_HOURS", "1")) + grace = timedelta(hours=grace_hours) + bucket = _env("S3_BUCKET", "messenger-attachments") + + s3 = boto3.client( + "s3", + endpoint_url=_env("S3_ENDPOINT"), + region_name=_env("S3_REGION", "us-east-1"), + aws_access_key_id=_env("S3_ACCESS_KEY_ID"), + aws_secret_access_key=_env("S3_SECRET_ACCESS_KEY"), + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + + s3_objects: dict[str, datetime] = {} + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket): + for obj in page.get("Contents", []): + s3_objects[obj["Key"]] = obj["LastModified"] + + log.info("S3 objects: %d", len(s3_objects)) + + db_keys: set[str] = set() + with psycopg.connect(_env("DATABASE_URL"), row_factory=dict_row) as conn: + with conn.cursor() as cur: + cur.execute("SELECT s3_key FROM attachments") + for row in cur.fetchall(): + db_keys.add(row["s3_key"]) + + log.info("DB attachments: %d", len(db_keys)) + + orphans = set(s3_objects) - db_keys + log.info("Orphans: %d", len(orphans)) + + now = datetime.now(timezone.utc) + deleted = 0 + skipped = 0 + for key in orphans: + age = now - s3_objects[key] + if age < grace: + log.info("skip %s (age %s < grace %s)", key, age, grace) + skipped += 1 + continue + try: + s3.delete_object(Bucket=bucket, Key=key) + log.info("deleted orphan: %s (age %s)", key, age) + deleted += 1 + except ClientError as exc: + log.error("failed to delete %s: %s", key, exc) + + log.info("done: deleted=%d, skipped=%d", deleted, skipped) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + log.exception("cleanup failed") + sys.exit(1) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..814bb90 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +docker compose --env-file .env restart postgres cache s3 broker + +i=0 +while [ $i -lt 120 ]; do + if docker compose --env-file .env exec -T broker rabbitmq-diagnostics -q ping > /dev/null 2>&1; then + break + fi + sleep 1 + i=$((i + 1)) +done + +docker compose --env-file .env restart api worker + +i=0 +while [ $i -lt 90 ]; do + if curl -sf http://localhost:8080/health/ready > /dev/null 2>&1; then + exit 0 + fi + sleep 1 + i=$((i + 1)) +done + +echo "stack did not become healthy in 90 seconds" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..9d99ee6 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID environment variable is required}" + +echo "Looking for API container with instance_id=$TARGET_INSTANCE_ID" + +found="" +for container in $(docker ps --format '{{.Names}}' | grep -E '^mtusi.*-api-[0-9]+$'); do + if docker logs "$container" 2>&1 | grep -q "instance_id=$TARGET_INSTANCE_ID"; then + found="$container" + break + fi +done + +if [ -z "$found" ]; then + echo "ERROR: no container found with instance_id=$TARGET_INSTANCE_ID" >&2 + exit 1 +fi + +echo "Stopping container: $found" +docker stop "$found" + +echo "Container $found stopped. Other replicas continue serving." +exit 0 \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..c89e0b3 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import hashlib +import os +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +import psycopg +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError, InvalidHashError + +from .db import transaction + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + _hasher.verify(password_hash, password) + return True + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _ttl_seconds() -> int: + return int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + + +def _generate_token() -> str: + return secrets.token_urlsafe(32) + + +def _token_hash(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: str) -> dict: + token = _generate_token() + ttl = _ttl_seconds() + expires_at = _now() + timedelta(seconds=ttl) + token_hash = _token_hash(token) + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at, created_at) + VALUES (%s, %s, %s, %s) + """, + (token_hash, user_id, expires_at, _now()), + ) + return {"token": token, "expires_at": expires_at} + + +def get_user_id_by_token(token: str) -> Optional[str]: + from . import cache + + token_hash = _token_hash(token) + + if cache.is_revoked(token_hash): + return None + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT user_id, expires_at + FROM sessions + WHERE token_hash = %s + """, + (token_hash,), + ) + row = cur.fetchone() + if row is None: + return None + if row["expires_at"] <= _now(): + return None + return str(row["user_id"]) + + +def revoke_session(token: str) -> bool: + from . import cache + + token_hash = _token_hash(token) + ttl = _ttl_seconds() + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT expires_at FROM sessions WHERE token_hash = %s", + (token_hash,), + ) + row = cur.fetchone() + if row is not None: + remaining = int((row["expires_at"] - _now()).total_seconds()) + if remaining > 0: + ttl = remaining + cur.execute( + "DELETE FROM sessions WHERE token_hash = %s", + (token_hash,), + ) + deleted = cur.rowcount > 0 + + if deleted: + cache.mark_revoked(token_hash, ttl) + return deleted \ No newline at end of file diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..61fa1d1 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +import logging +import os +import uuid +from typing import Callable, Optional + +import pika +from pika.exceptions import AMQPConnectionError + +from opentelemetry import trace +from opentelemetry.propagate import inject as otel_inject, extract as otel_extract +from opentelemetry.trace import SpanKind, Link + +_tracer = trace.get_tracer("messenger.broker") + +log = logging.getLogger("messenger.broker") + +QUEUE = "attachment.process.v1" +DLQ = "attachment.process.v1.dlq" + + +def _url() -> str: + url = os.environ.get("BROKER_URL") + if not url: + raise RuntimeError("BROKER_URL environment variable is required") + return url + + +def _connect() -> pika.BlockingConnection: + params = pika.URLParameters(_url()) + params.heartbeat = 30 + params.blocked_connection_timeout = 5 + return pika.BlockingConnection(params) + + +def _declare(channel: pika.channel.Channel) -> None: + # DLQ + channel.queue_declare(queue=DLQ, durable=True) + # Основная очередь с DLX на DLQ + channel.queue_declare( + queue=QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": DLQ, + "x-max-priority": 5, + }, + ) + + +def publish_job(job: dict) -> None: + body = json.dumps(job, ensure_ascii=False).encode("utf-8") + + parent_ctx = None + stored_tp = job.get("_traceparent") + stored_ts = job.get("_tracestate") + if stored_tp: + parent_ctx = otel_extract({"traceparent": stored_tp, + **({"tracestate": stored_ts} if stored_ts else {})}) + + with _tracer.start_as_current_span( + "job.publish", + context=parent_ctx, + kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "messaging.operation": "publish", + "messaging.message.id": job["job_id"], + "job.type": job.get("type", "attachment.process.v1"), + "job.attempt": job.get("attempt", 1), + "job.attachment_id": job.get("attachment_id", ""), + }, + ) as span: + headers: dict = {} + otel_inject(headers) + + try: + conn = _connect() + except AMQPConnectionError as exc: + span.record_exception(exc) + span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc))) + raise RuntimeError(f"broker unavailable: {exc}") from exc + try: + ch = conn.channel() + _declare(ch) + ch.confirm_delivery() + clean_job = {k: v for k, v in job.items() if not k.startswith("_")} + clean_body = json.dumps(clean_job, ensure_ascii=False).encode("utf-8") + ch.basic_publish( + exchange="", + routing_key=QUEUE, + body=clean_body, + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + message_id=job["job_id"], + headers=headers, + ), + mandatory=True, + ) + log.info("published job %s for attachment %s", + job["job_id"], job["attachment_id"]) + finally: + conn.close() + + +def consume(handler: Callable[[dict], None], prefetch: int = 2) -> None: + conn = _connect() + ch = conn.channel() + _declare(ch) + ch.basic_qos(prefetch_count=prefetch) + + def callback(ch_, method, properties, body): + try: + job = json.loads(body) + except Exception as exc: + log.error("invalid job payload: %s", exc) + ch_.basic_nack(method.delivery_tag, requeue=False) + return + + headers = properties.headers or {} + parent_ctx = otel_extract(headers) if headers.get("traceparent") else None + + try: + handler(job, parent_ctx=parent_ctx) + ch_.basic_ack(method.delivery_tag) + except Exception as exc: + log.exception("job failed: %s", exc) + ch_.basic_nack(method.delivery_tag, requeue=False) + + ch.basic_consume(queue=QUEUE, on_message_callback=callback) + log.info("worker started, queue=%s prefetch=%d", QUEUE, prefetch) + try: + ch.start_consuming() + except KeyboardInterrupt: + ch.stop_consuming() + finally: + conn.close() + +def publish_dlq(job: dict, error: str) -> None: + with _tracer.start_as_current_span( + "job.dlq_publish", + kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": DLQ, + "messaging.operation": "publish", + "messaging.message.id": job.get("job_id", ""), + }, + ): + body = json.dumps( + {"job": job, "error": error, "failed_at": job.get("created_at")}, + ensure_ascii=False, + ).encode("utf-8") + headers: dict = {} + otel_inject(headers) + conn = _connect() + try: + ch = conn.channel() + ch.queue_declare(queue=DLQ, durable=True) + ch.basic_publish( + exchange="", + routing_key=DLQ, + body=body, + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + headers=headers, + ), + ) + finally: + conn.close() \ No newline at end of file diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 0000000..2b10432 --- /dev/null +++ b/src/cache.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import logging +import os +from typing import Optional + +import redis + +log = logging.getLogger("messenger.cache") + +_client: Optional[redis.Redis] = None +_failures: int = 0 +_MAX_FAILURES_BEFORE_DISABLE: int = 3 + + +def _cache_url() -> Optional[str]: + return os.environ.get("CACHE_URL") + + +def init_cache() -> None: + global _client, _failures + url = _cache_url() + if not url: + log.warning("CACHE_URL not set; running without cache") + _client = None + return + try: + client = redis.Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=1.0, + socket_timeout=1.0, + ) + client.ping() + _client = client + _failures = 0 + log.info("cache connected") + except Exception as exc: + log.warning("cache unavailable at startup: %s; running in degraded mode", exc) + _client = None + + +def close_cache() -> None: + global _client + if _client is not None: + try: + _client.close() + except Exception: + pass + _client = None + + +def _note_failure() -> None: + global _client, _failures + _failures += 1 + if _failures >= _MAX_FAILURES_BEFORE_DISABLE and _client is not None: + log.warning( + "cache disabled after %d consecutive failures; running in degraded mode", + _failures, + ) + try: + _client.close() + except Exception: + pass + _client = None + + +def _note_success() -> None: + global _failures + _failures = 0 + + +def is_available() -> bool: + return _client is not None + + +def mark_revoked(token_hash: str, ttl_seconds: int) -> None: + if _client is None: + log.warning( + "cache unavailable; revocation not recorded in Redis " + "(Postgres is the source of truth, next request will 401)" + ) + return + if ttl_seconds <= 0: + return + try: + _client.setex(f"revoked:{token_hash}", ttl_seconds, "1") + _note_success() + except Exception as exc: + log.warning("mark_revoked failed: %s", exc) + _note_failure() + + +def is_revoked(token_hash: str) -> bool: + if _client is None: + return False + try: + exists = _client.exists(f"revoked:{token_hash}") + _note_success() + return exists == 1 + except Exception as exc: + log.warning("is_revoked failed: %s", exc) + _note_failure() + return False \ No newline at end of file diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..d0455b6 --- /dev/null +++ b/src/db.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import os +from contextlib import contextmanager +from typing import Iterator + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +_pool: ConnectionPool | None = None + + +def _database_url() -> str: + url = os.environ.get("DATABASE_URL") + if not url: + raise RuntimeError("DATABASE_URL environment variable is required") + return url + + +from psycopg_pool import ConnectionPool + +def init_pool(min_size: int = 1, max_size: int = 10) -> None: + global _pool + if _pool is not None: + return + + dsn = _database_url() + sep = "&" if "?" in dsn else "?" + dsn_with_opts = ( + dsn + + sep + + "options=" + + "-c%20statement_timeout%3D5000" + + "%20-c%20lock_timeout%3D3000" + + "%20-c%20idle_in_transaction_session_timeout%3D10000" + ) + + _pool = ConnectionPool( + conninfo=dsn_with_opts, + min_size=min_size, + max_size=max_size, + kwargs={"row_factory": dict_row}, + open=True, + timeout=5.0, + check=ConnectionPool.check_connection, + max_lifetime=120, + max_idle=60, + ) + + +def close_pool() -> None: + global _pool + if _pool is not None: + _pool.close() + _pool = None + + +@contextmanager +def connection() -> Iterator[psycopg.Connection]: + if _pool is None: + raise RuntimeError("Connection pool is not initialized") + with _pool.connection() as conn: + yield conn + + +@contextmanager +def transaction() -> Iterator[psycopg.Connection]: + with connection() as conn: + with conn.transaction(): + yield conn \ No newline at end of file diff --git a/src/errors.py b/src/errors.py new file mode 100755 index 0000000..fbcf1f0 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import uuid +from typing import Callable + +from fastapi import Request +from fastapi.responses import JSONResponse +from starlette.middleware.base import BaseHTTPMiddleware + +from .instance import INSTANCE_ID +from .store import ApiError + + +REQUEST_ID_HEADER = "X-Request-Id" + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + rid = request.headers.get(REQUEST_ID_HEADER) or uuid.uuid4().hex + request.state.request_id = rid + response = await call_next(request) + response.headers[REQUEST_ID_HEADER] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +def _envelope(code: str, message: str, request_id: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def _rid(request: Request) -> str: + return getattr(request.state, "request_id", None) or uuid.uuid4().hex + + +def _with_headers(response: JSONResponse, request: Request, status: int) -> JSONResponse: + rid = _rid(request) + response.headers[REQUEST_ID_HEADER] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + if status == 401: + scheme = "Bearer" + if request.url.path == "/api/v1/auth/sessions": + scheme = "Basic" + response.headers["WWW-Authenticate"] = f'{scheme} realm="messenger"' + return response + + +async def api_error_handler(request: Request, exc: Exception) -> JSONResponse: + assert isinstance(exc, ApiError) + response = JSONResponse( + status_code=exc.status, + content=_envelope(exc.code, exc.message, _rid(request)), + ) + return _with_headers(response, request, exc.status) + + +async def validation_handler(request: Request, exc: Exception) -> JSONResponse: + response = JSONResponse( + status_code=400, + content=_envelope("invalid_request", "Request validation failed", _rid(request)), + ) + return _with_headers(response, request, 400) + + +async def json_decode_handler(request: Request, exc: Exception) -> JSONResponse: + response = JSONResponse( + status_code=400, + content=_envelope("invalid_request", "Malformed JSON", _rid(request)), + ) + return _with_headers(response, request, 400) + + +async def unhandled_handler(request: Request, exc: Exception) -> JSONResponse: + import logging + logging.getLogger("messenger").exception( + "unhandled error on %s %s", request.method, request.url.path + ) + response = JSONResponse( + status_code=500, + content=_envelope("internal_error", "Internal error", _rid(request)), + ) + return _with_headers(response, request, 500) \ No newline at end of file diff --git a/src/instance.py b/src/instance.py new file mode 100644 index 0000000..701ad56 --- /dev/null +++ b/src/instance.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import os +import socket +import uuid + +INSTANCE_ID: str = ( + os.environ.get("INSTANCE_ID") + or socket.gethostname() + or uuid.uuid4().hex[:8] +) \ No newline at end of file diff --git a/src/logging_config.py b/src/logging_config.py new file mode 100644 index 0000000..fff85b1 --- /dev/null +++ b/src/logging_config.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +import logging +import sys +from typing import Any + +from opentelemetry import trace + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + import datetime as _dt + ts = _dt.datetime.fromtimestamp(record.created, tz=_dt.timezone.utc) + payload: dict[str, Any] = { + "timestamp": ts.isoformat(timespec="milliseconds").replace("+00:00", "Z"), + "severity": record.levelname, + "logger": record.name, + "event": record.getMessage(), + } + + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + if ctx is not None and ctx.is_valid: + payload["trace_id"] = format(ctx.trace_id, "032x") + payload["span_id"] = format(ctx.span_id, "016x") + + for key in ("request_id", "instance_id", "service_name", "attempt", + "job_id", "attachment_id", "event_kind"): + if hasattr(record, key): + payload[key] = getattr(record, key) + + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + + return json.dumps(payload, ensure_ascii=False, default=str) + + +def configure_logging(service_name: str, level: int = logging.INFO) -> None: + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + + handler = logging.StreamHandler(stream=sys.stdout) + handler.setFormatter(JsonFormatter()) + root.addHandler(handler) + root.setLevel(level) + + for noisy in ("urllib3", "botocore", "boto3", "pika", "uvicorn.access"): + logging.getLogger(noisy).setLevel(logging.WARNING) + + logging.getLogger().info( + "logging configured", extra={"service_name": service_name} + ) \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100755 index 0000000..8500d1b --- /dev/null +++ b/src/main.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import logging +import os +import re +from contextlib import asynccontextmanager +from typing import Optional +import uuid +from uuid import UUID + +from fastapi import Depends, FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import auth +from .db import close_pool, init_pool +from . import storage +from . import broker, storage +from .errors import ( + RequestIdMiddleware, + _envelope, + _rid, + api_error_handler, + json_decode_handler, + unhandled_handler, + validation_handler, +) +from .schemas import ( + Chat, + ChatPage, + CreateChat, + CreateMessage, + CreateUser, + Health, + Message, + MessagePage, + Session, + User, + Attachment, +) +from .store import ( + ApiError, + BadRequest, + Conflict, + NotFound, + PayloadTooLarge, + Unavailable, + UnsupportedMediaType, + Unauthorized, + store, + iso, +) +from .cache import close_cache, init_cache + +from .instance import INSTANCE_ID + +from . import otel as otel_module +from .logging_config import configure_logging +import time as _time +from prometheus_client import ( + Counter as _PromCounter, + Histogram as _PromHistogram, + Gauge as _PromGauge, + CollectorRegistry as _PromRegistry, + generate_latest as _prom_generate, + CONTENT_TYPE_LATEST as _PROM_CONTENT_TYPE, +) + +configure_logging("messenger-api") +log = logging.getLogger("messenger") + + +_PROM_REGISTRY = _PromRegistry() +_HTTP_REQUESTS = _PromCounter( + "http_requests_total", + "HTTP requests by method, route template, status class", + ["method", "route", "status_class"], + registry=_PROM_REGISTRY, +) +_HTTP_DURATION = _PromHistogram( + "http_request_duration_seconds", + "HTTP request duration in seconds", + ["method", "route", "status_class"], + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0), + registry=_PROM_REGISTRY, +) +_PROM_GAUGE_POOL_USED = _PromGauge( + "db_pool_used", + "DB pool connections currently in use", + registry=_PROM_REGISTRY, +) +_PROM_GAUGE_POOL_AVAILABLE = _PromGauge( + "db_pool_available", + "DB pool connections currently available", + registry=_PROM_REGISTRY, +) +_PROM_GAUGE_POOL_WAITING = _PromGauge( + "db_pool_waiting", + "Requests waiting for a DB connection", + registry=_PROM_REGISTRY, +) + + +def _refresh_db_pool_metrics(): + from . import db as _db + pool = _db._pool + if pool is None: + return + try: + stats = pool.get_stats() + except Exception: + return + used = max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)) + _PROM_GAUGE_POOL_USED.set(used) + _PROM_GAUGE_POOL_AVAILABLE.set(stats.get("pool_available", 0)) + _PROM_GAUGE_POOL_WAITING.set(stats.get("requests_waiting", 0)) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + otel_module.setup_otel("messenger-api") + otel_module.instrument_libraries(psycopg=True, redis=True, botocore=True) + init_pool() + init_cache() + storage.init_storage() + log.info("instance_id=%s", INSTANCE_ID) + log.info("db pool, cache, storage initialized") + try: + yield + finally: + close_cache() + close_pool() + log.info("db pool and cache closed") + + +app = FastAPI(title="MTUSI Messenger lab2", version="1.2.0", lifespan=lifespan) +app.add_middleware(RequestIdMiddleware) + +otel_module.instrument_fastapi(app) + +from starlette.middleware.base import BaseHTTPMiddleware + + +class MetricsMiddleware(BaseHTTPMiddleware): + + async def dispatch(self, request: Request, call_next): + if request.url.path in ("/health/live", "/health/ready", "/metrics"): + return await call_next(request) + + start = _time.monotonic() + response = await call_next(request) + elapsed = _time.monotonic() - start + + route = request.scope.get("route") + route_template = getattr(route, "path", None) or request.url.path + status_class = f"{response.status_code // 100}xx" + + _HTTP_REQUESTS.labels( + method=request.method, route=route_template, status_class=status_class + ).inc() + _HTTP_DURATION.labels( + method=request.method, route=route_template, status_class=status_class + ).observe(elapsed) + + return response + +app.add_middleware(MetricsMiddleware) + +app.add_exception_handler(ApiError, api_error_handler) +app.add_exception_handler(RequestValidationError, validation_handler) +app.add_exception_handler(Exception, unhandled_handler) + + +@app.exception_handler(StarletteHTTPException) +async def _http_exc(request: Request, exc: StarletteHTTPException): + code = "not_found" if exc.status_code == 404 else "invalid_request" + return JSONResponse( + status_code=exc.status_code, + content=_envelope(code, str(exc.detail), _rid(request)), + ) + +# ---------- утилиты валидации ---------- + +def valid_uuid(value: str, name: str = "id") -> str: + try: + UUID(value) + except (ValueError, TypeError, AttributeError): + raise BadRequest(f"Invalid {name}") + return value + +# ---------- зависимости ---------- + +def _parse_basic(header_value: str) -> tuple[str, str]: + import base64 + if not header_value.lower().startswith("basic "): + raise Unauthorized("Basic authentication required") + try: + raw = base64.b64decode(header_value.split(" ", 1)[1], validate=True) + decoded = raw.decode("utf-8") + username, _, password = decoded.partition(":") + if not username: + raise ValueError("empty username") + return username, password + except Exception: + raise Unauthorized("Invalid Basic credentials") from None + + +def _bearer_token(authorization: Optional[str]) -> str: + if not authorization: + raise Unauthorized("Missing Authorization header") + parts = authorization.split(" ", 1) + if len(parts) != 2 or parts[0].lower() != "bearer": + raise Unauthorized("Bearer token required") + token = parts[1].strip() + if not token: + raise Unauthorized("Empty bearer token") + return token + + +def current_user_id( + authorization: Optional[str] = Header(default=None), +) -> str: + token = _bearer_token(authorization) + user_id = auth.get_user_id_by_token(token) + if user_id is None: + raise Unauthorized("Invalid or expired token") + return user_id + + +# ---------- health ---------- + +@app.get("/health/live", response_model=Health) +def liveness(): + return {"status": "ok"} + + +@app.get("/health/ready", response_model=Health) +def readiness(request: Request): + try: + from .db import transaction + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + except Exception: + rid = _rid(request) + response = JSONResponse( + status_code=503, + content=_envelope("unavailable", "Database unavailable", rid), + ) + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + try: + storage.check_bucket() + except Exception: + rid = _rid(request) + response = JSONResponse( + status_code=503, + content=_envelope("unavailable", "S3 unavailable", rid), + ) + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + return {"status": "ok"} + +@app.get("/metrics", include_in_schema=False) +def metrics_endpoint(): + _refresh_db_pool_metrics() + data = _prom_generate(_PROM_REGISTRY) + return Response(content=data, media_type=_PROM_CONTENT_TYPE) + +# ---------- users ---------- + +@app.post("/api/v1/users", status_code=201, response_model=User) +def register_user(body: CreateUser): + password_hash = auth.hash_password(body.password) + return store.create_user(body.username, body.display_name, password_hash) + + +@app.get("/api/v1/users/me", response_model=User) +def me(user_id: str = Depends(current_user_id)): + user = store.get_user(user_id) + if user is None: + raise Unauthorized("User not found") + return user + + +# ---------- auth ---------- + +@app.post("/api/v1/auth/sessions", status_code=201, response_model=Session) +def create_session(authorization: Optional[str] = Header(default=None)): + username, password = _parse_basic(authorization or "") + row = store.get_user_by_username(username) + if row is None or not auth.verify_password(password, row["password_hash"]): + raise Unauthorized("Invalid credentials") + session = auth.create_session(str(row["id"])) + user = store.get_user(str(row["id"])) + if user is None: + raise Unauthorized("User not found") + return { + "token": session["token"], + "token_type": "Bearer", + "expires_at": iso(session["expires_at"]), + "user": user, + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(authorization: Optional[str] = Header(default=None)): + token = _bearer_token(authorization) + user_id = auth.get_user_id_by_token(token) + if user_id is None: + raise Unauthorized("Invalid or expired token") + auth.revoke_session(token) + return Response(status_code=204) + + +# ---------- chats ---------- + +@app.post("/api/v1/chats", status_code=201, response_model=Chat) +def create_chat(body: CreateChat, user_id: str = Depends(current_user_id)): + return store.create_chat(body.title, body.member_ids, user_id) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), + user_id: str = Depends(current_user_id), +): + items, next_cursor = store.list_chats(user_id, limit, cursor) + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(chat_id: str, user_id: str = Depends(current_user_id)): + chat_id = valid_uuid(chat_id, "chat_id") + chat = store.get_chat(chat_id, user_id) + if chat is None: + raise NotFound() + return chat + + +# ---------- messages ---------- + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201, response_model=Message) +def create_message( + chat_id: str, + body: CreateMessage, + user_id: str = Depends(current_user_id), + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), +): + chat_id = valid_uuid(chat_id, "chat_id") + if not idempotency_key: + raise BadRequest("Idempotency-Key header is required") + if not re.match(r"^[A-Za-z0-9._:-]{1,128}$", idempotency_key): + raise BadRequest("Invalid Idempotency-Key format") + + message, status = store.add_message( + chat_id, user_id, body.text, idempotency_key, body.attachment_ids + ) + if status != 201: + raise Conflict(f"Idempotency error: {status}") + return message + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +def list_messages( + chat_id: str, + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), + user_id: str = Depends(current_user_id), +): + chat_id = valid_uuid(chat_id, "chat_id") + items, next_cursor = store.list_messages(chat_id, user_id, limit, cursor) + return {"items": items, "next_cursor": next_cursor} + + +# ---------- attachments (lab4) ---------- + +MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +ALLOWED_MIMES = {"image/png", "image/jpeg"} + +from PIL import Image as _PILImage +_PILImage.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + + +import unicodedata + +def _validate_filename(filename: str) -> None: + if not filename: + raise BadRequest("filename is required") + if len(filename) > 255: + raise BadRequest("filename too long") + for ch in filename: + if ch in ("/", "\\"): + raise BadRequest("filename contains forbidden characters") + if unicodedata.category(ch) == "Cc": + raise BadRequest("filename contains control characters") + + +def _sniff_mime(data: bytes) -> str: + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + raise UnsupportedMediaType("Only PNG and JPEG are supported") + + +def _validate_image(data: bytes) -> tuple[int, int]: + from PIL import Image + import io + try: + img = Image.open(io.BytesIO(data)) + w, h = img.size + if w * h > MAX_IMAGE_PIXELS: + raise PayloadTooLarge( + f"Image too large: {w*h} pixels (max {MAX_IMAGE_PIXELS})" + ) + img.load() + except PayloadTooLarge: + raise + except Image.DecompressionBombError: + raise PayloadTooLarge( + f"Image too large (decompression bomb protection)" + ) from None + except Exception: + raise UnsupportedMediaType("Image is corrupted or unsupported") from None + return w, h + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + status_code=202, + response_model=Attachment, +) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + user_id: str = Depends(current_user_id), +): + chat_id = valid_uuid(chat_id, "chat_id") + _validate_filename(filename) + + content_length = request.headers.get("content-length") + if content_length is not None: + try: + if int(content_length) > MAX_UPLOAD_BYTES: + raise PayloadTooLarge( + f"File too large: {content_length} bytes (max {MAX_UPLOAD_BYTES})" + ) + except ValueError: + raise BadRequest("Invalid Content-Length") from None + + client_ct = (request.headers.get("content-type") or "").split(";")[0].strip().lower() + if client_ct not in ALLOWED_MIMES: + raise UnsupportedMediaType( + f"Content-Type must be image/png or image/jpeg, got {client_ct!r}" + ) + + body = bytearray() + async for chunk in request.stream(): + body.extend(chunk) + if len(body) > MAX_UPLOAD_BYTES: + raise PayloadTooLarge( + f"File too large: > {MAX_UPLOAD_BYTES} bytes" + ) + data = bytes(body) + + real_mime = _sniff_mime(data) + + _validate_image(data) + + ext = "png" if real_mime == "image/png" else "jpg" + staging_key = f"staging/{uuid.uuid4().hex}.{ext}" + try: + sha256 = storage.put_object(staging_key, data, real_mime) + except Exception as exc: + import logging + logging.getLogger("messenger").warning("S3 staging upload failed: %s", exc) + raise Unavailable("S3 unavailable") from exc + + try: + attachment = store.create_attachment( + chat_id=chat_id, owner_id=user_id, filename=filename, + content_type=real_mime, size_bytes=len(data), + sha256=sha256, s3_key=staging_key, staging_key=staging_key, + status="queued", + ) + except Exception: + storage.delete_object(staging_key) + raise + + job = store.create_job(attachment["id"]) + log.info("job %s created (pending), relay will publish", job["job_id"]) + + return attachment + + +@app.get( + "/api/v1/attachments/{attachment_id}", + response_model=Attachment, +) +def get_attachment( + attachment_id: str, + user_id: str = Depends(current_user_id), +): + attachment_id = valid_uuid(attachment_id, "attachment_id") + att = store.get_attachment(attachment_id, user_id) + if att is None: + raise NotFound() + att.pop("s3_key", None) + return att + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + variant: str = Query(default="original"), + user_id: str = Depends(current_user_id), +): + attachment_id = valid_uuid(attachment_id, "attachment_id") + att = store.get_attachment(attachment_id, user_id) + if att is None: + raise NotFound() + if att["status"] != "ready": + raise Conflict(f"Attachment not ready: {att['status']}") + variants_map = { + "thumbnail": ("thumbnails/{}.jpg", "image/jpeg"), + "crop": ("crops/{}.jpg", "image/jpeg"), + "optimized": ("optimized/{}.webp", "image/webp"), + } + if variant == "original": + s3_key = att["s3_key"] + media_type = att["content_type"] + elif variant in variants_map: + found = [v for v in att["variants"] if v["name"] == variant] + if not found: + raise NotFound() + pattern, media_type = variants_map[variant] + s3_key = pattern.format(attachment_id) + else: + raise NotFound() + data = storage.get_object(s3_key) + return Response(content=data, media_type=media_type) + +def main() -> None: + import uvicorn + + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8080")) + uvicorn.run("src.main:app", host=host, port=port, log_level="info") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/otel.py b/src/otel.py new file mode 100644 index 0000000..74eaf5a --- /dev/null +++ b/src/otel.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import atexit +import logging +import os +import socket +import uuid +from typing import Iterable + +from opentelemetry import metrics, trace +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import ( + DEPLOYMENT_ENVIRONMENT, + SERVICE_INSTANCE_ID, + SERVICE_NAME, + SERVICE_VERSION, + Resource, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry import _logs as otel_logs + +_initialized = False +_tracer_provider: TracerProvider | None = None +_meter_provider: MeterProvider | None = None +_logger_provider: LoggerProvider | None = None + + +def _endpoint() -> str: + return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318") + + +def _service_name() -> str: + return os.environ.get("OTEL_SERVICE_NAME", "messenger-api") + + +def _service_version() -> str: + return os.environ.get("SERVICE_VERSION", "1.6.0") + + +def _deployment_env() -> str: + return os.environ.get("OTEL_RESOURCE_ATTRIBUTES_DEPLOYMENT_ENV", "lab") + + +def _sampler_ratio() -> float: + try: + return float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0")) + except ValueError: + return 1.0 + + +def _build_resource(service_name: str, extra: dict | None = None) -> Resource: + instance = os.environ.get("INSTANCE_ID") or socket.gethostname() or uuid.uuid4().hex[:8] + attrs = { + SERVICE_NAME: service_name, + SERVICE_VERSION: _service_version(), + SERVICE_INSTANCE_ID: instance, + DEPLOYMENT_ENVIRONMENT: _deployment_env(), + "service.namespace": "messenger", + } + if extra: + attrs.update(extra) + return Resource.create(attrs) + + +def setup_otel( + service_name: str | None = None, + extra_resource: dict | None = None, + *, + enable_traces: bool = True, + enable_metrics: bool = True, + enable_logs: bool = True, +) -> tuple[trace.Tracer, metrics.Meter]: + global _initialized, _tracer_provider, _meter_provider, _logger_provider + + if _initialized: + return ( + trace.get_tracer(_service_name()), + metrics.get_meter(_service_name()), + ) + + svc = service_name or _service_name() + resource = _build_resource(svc, extra_resource) + + if enable_traces: + tracer_provider = TracerProvider( + resource=resource, + sampler=ParentBased(TraceIdRatioBased(_sampler_ratio())), + ) + span_exporter = OTLPSpanExporter(endpoint=f"{_endpoint()}/v1/traces") + tracer_provider.add_span_processor( + BatchSpanProcessor( + span_exporter, + max_queue_size=2048, + max_export_batch_size=512, + schedule_delay_millis=5000, + export_timeout_millis=10000, + ) + ) + trace.set_tracer_provider(tracer_provider) + _tracer_provider = tracer_provider + + if enable_metrics: + metric_exporter = OTLPMetricExporter(endpoint=f"{_endpoint()}/v1/metrics") + reader = PeriodicExportingMetricReader( + metric_exporter, + export_interval_millis=10000, + export_timeout_millis=10000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(meter_provider) + _meter_provider = meter_provider + + if enable_logs: + logger_provider = LoggerProvider(resource=resource) + log_exporter = OTLPLogExporter(endpoint=f"{_endpoint()}/v1/logs") + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + log_exporter, + max_queue_size=2048, + max_export_batch_size=512, + schedule_delay_millis=5000, + export_timeout_millis=10000, + ) + ) + otel_logs.set_logger_provider(logger_provider) + _logger_provider = logger_provider + + root = logging.getLogger() + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + root.addHandler(otel_handler) + + _initialized = True + logging.getLogger(__name__).info( + "otel initialized", + extra={"service_name": svc, "event_kind": "otel_init"}, + ) + atexit.register(shutdown_otel) + return trace.get_tracer(svc), metrics.get_meter(svc) + + +def shutdown_otel() -> None: + global _initialized, _tracer_provider, _meter_provider, _logger_provider + if not _initialized: + return + try: + if _tracer_provider is not None: + _tracer_provider.shutdown() + if _meter_provider is not None: + _meter_provider.shutdown() + if _logger_provider is not None: + _logger_provider.shutdown() + except Exception: + pass + _initialized = False + + +def instrument_fastapi(app) -> None: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + FastAPIInstrumentor.instrument_app( + app, + tracer_provider=trace.get_tracer_provider(), + excluded_urls="health/live,health/ready,metrics", # не засоряем трейсы + ) + + +def instrument_libraries( + *, + psycopg: bool = False, + redis: bool = False, + botocore: bool = False, +) -> None: + if psycopg: + try: + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + PsycopgInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) + except Exception: + pass + if redis: + try: + from opentelemetry.instrumentation.redis import RedisInstrumentor + RedisInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) + except Exception: + pass + if botocore: + try: + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + BotocoreInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) + except Exception: + pass \ No newline at end of file diff --git a/src/relay.py b/src/relay.py new file mode 100644 index 0000000..b5d3f97 --- /dev/null +++ b/src/relay.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import logging +import time +import os + +from . import broker +from .db import close_pool, init_pool, transaction +from .store import iso + +from . import otel as otel_module +from .logging_config import configure_logging +from prometheus_client import ( + Gauge as _PromGauge, + CollectorRegistry as _PromRegistry, + generate_latest as _prom_generate, + CONTENT_TYPE_LATEST as _PROM_CONTENT_TYPE, +) + +configure_logging("messenger-relay") +log = logging.getLogger("messenger.relay") + +_PROM_REGISTRY = _PromRegistry() +_JOBS_OLDEST_AGE = _PromGauge( + "jobs_oldest_pending_age_seconds", + "Age of the oldest job in pending/processing state (seconds)", + registry=_PROM_REGISTRY, +) +_JOBS_PENDING = _PromGauge( + "jobs_pending_count", + "Number of jobs currently in 'pending' outbox state", + registry=_PROM_REGISTRY, +) +_JOBS_PROCESSING = _PromGauge( + "jobs_processing_count", + "Number of jobs currently in 'processing' state (leased)", + registry=_PROM_REGISTRY, +) + + +def _publish_pending() -> int: + published = 0 + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT job_id, attachment_id, pipeline_version, attempt, + created_at, traceparent, tracestate + FROM jobs + WHERE status = 'pending' + ORDER BY created_at + LIMIT 10 + FOR UPDATE SKIP LOCKED + """ + ) + jobs = cur.fetchall() + for job in jobs: + job_dict = { + "schema_version": 1, + "job_id": str(job["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(job["attachment_id"]), + "pipeline_version": int(job["pipeline_version"]), + "attempt": int(job["attempt"]), + "created_at": iso(job["created_at"]), + "_traceparent": job["traceparent"], + "_tracestate": job["tracestate"], + } + try: + broker.publish_job(job_dict) + cur.execute( + "UPDATE jobs SET status='published', published_at=NOW() WHERE job_id=%s", + (job["job_id"],), + ) + published += 1 + except Exception as exc: + log.warning("publish failed for %s: %s", job["job_id"], exc) + return published + + +def _reclaim_expired() -> int: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE jobs + SET status = 'pending', lease_until = NULL + WHERE status = 'processing' + AND lease_until < NOW() + RETURNING job_id + """ + ) + rows = cur.fetchall() + return len(rows) + + +def _refresh_gauges() -> None: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + COALESCE(EXTRACT(EPOCH FROM NOW() - MIN(created_at)), 0) + FROM jobs WHERE status IN ('pending', 'processing') + """ + ) + age = float(cur.fetchone()[0] if cur.rowcount else 0.0) + cur.execute("SELECT COUNT(*) FROM jobs WHERE status='pending'") + pending = int(cur.fetchone()[0]) + cur.execute("SELECT COUNT(*) FROM jobs WHERE status='processing'") + processing = int(cur.fetchone()[0]) + _JOBS_OLDEST_AGE.set(max(0.0, age)) + _JOBS_PENDING.set(pending) + _JOBS_PROCESSING.set(processing) + + +def main() -> None: + otel_module.setup_otel("messenger-relay") + otel_module.instrument_libraries(psycopg=True) + + init_pool() + + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + class MetricsHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/metrics": + data = _prom_generate(_PROM_REGISTRY) + self.send_response(200) + self.send_header("Content-Type", _PROM_CONTENT_TYPE) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + elif self.path == "/health/live": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + else: + self.send_response(404) + self.end_headers() + + def log_message(self, *args): + pass + + metrics_port = int(os.environ.get("RELAY_METRICS_PORT", "9101")) + httpd = ThreadingHTTPServer(("0.0.0.0", metrics_port), MetricsHandler) + threading.Thread(target=httpd.serve_forever, daemon=True, name="metrics-http").start() + log.info("relay metrics on :%d/metrics", metrics_port) + + log.info("relay started") + try: + while True: + try: + n = _publish_pending() + if n: + log.info("published %d pending jobs", n) + _refresh_gauges() + except Exception as exc: + log.exception("relay iteration failed: %s", exc) + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + otel_module.shutdown_otel() + close_pool() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/schemas.py b/src/schemas.py new file mode 100755 index 0000000..c2d9911 --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import re +import uuid +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=False) + + +# ---------- Users ---------- + +class CreateUser(StrictModel): + username: str = Field(min_length=3, max_length=32) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def _username(cls, v: str) -> str: + if not USERNAME_RE.match(v): + raise ValueError("username must match [a-z0-9_]{3,32}") + return v + + @field_validator("display_name") + @classmethod + def _display_name(cls, v: str) -> str: + if not v.strip(): + raise ValueError("display_name must not be blank") + if len(v) > 100: + raise ValueError("display_name too long") + return v + + +class User(StrictModel): + id: str + username: str + display_name: str + created_at: str + + +# ---------- Chats ---------- + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: List[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def _title(cls, v: str) -> str: + if not v.strip(): + raise ValueError("title must not be blank") + return v + + @field_validator("member_ids") + @classmethod + def _members(cls, v: List[str]) -> List[str]: + if len(set(v)) != len(v): + raise ValueError("member_ids must be unique") + for item in v: + try: + uuid.UUID(item) + except ValueError as exc: + raise ValueError("member_ids must be UUIDs") from exc + return v + + +class Chat(StrictModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class ChatPage(StrictModel): + items: List[Chat] + next_cursor: Optional[str] = None + + +# ---------- Attachments (lab4) ---------- + +class Variant(StrictModel): + name: str + content_type: str + size_bytes: int = Field(ge=1) + width: int = Field(ge=1) + height: int = Field(ge=1) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + +class Attachment(StrictModel): + id: str + chat_id: str + owner_id: str + filename: str = Field(min_length=1, max_length=255) + content_type: str # image/png или image/jpeg + size_bytes: int = Field(ge=1, le=10485760) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + status: str # 'ready' в lab4 + variants: List[Variant] = Field(default_factory=list) + error_code: Optional[str] = None + created_at: str + updated_at: str + + +# ---------- Messages ---------- + +class CreateMessage(StrictModel): + text: str = Field(default="", max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def _text(cls, v: str) -> str: + if len(v) > 4000: + raise ValueError("text too long") + return v + + @field_validator("attachment_ids") + @classmethod + def _attachments(cls, v: List[str]) -> List[str]: + if len(set(v)) != len(v): + raise ValueError("attachment_ids must be unique") + for item in v: + try: + uuid.UUID(item) + except ValueError as exc: + raise ValueError("attachment_ids must be UUIDs") from exc + return v + + @model_validator(mode="after") + def _text_or_attachment(self) -> "CreateMessage": + has_text = bool(self.text and self.text.strip()) + has_attachment = bool(self.attachment_ids) + if not has_text and not has_attachment: + raise ValueError("text must be non-blank or attachment_ids must be non-empty") + return self + + +class Message(StrictModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: List[str] = Field(default_factory=list) + + +class MessagePage(StrictModel): + items: List[Message] + next_cursor: Optional[str] = None + + +# ---------- Sessions ---------- + +class Session(StrictModel): + token: str = Field(min_length=32) + token_type: str = Field(default="Bearer") + expires_at: str + user: User + + +# ---------- Misc ---------- + +class Health(StrictModel): + status: str = "ok" + + +class ErrorBody(StrictModel): + code: str + message: str + request_id: str + + +class ErrorEnvelope(StrictModel): + error: ErrorBody \ No newline at end of file diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..24eff09 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import hashlib +import io +import logging +import os +import uuid +from typing import Optional + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError + +log = logging.getLogger("messenger.storage") + +_client = None + + +def _env(name: str, default: Optional[str] = None) -> str: + value = os.environ.get(name, default) + if value is None: + raise RuntimeError(f"Environment variable {name} is required") + return value + + +def init_storage() -> None: + global _client + if _client is not None: + return + _client = boto3.client( + "s3", + endpoint_url=_env("S3_ENDPOINT"), + region_name=_env("S3_REGION", "us-east-1"), + aws_access_key_id=_env("S3_ACCESS_KEY_ID"), + aws_secret_access_key=_env("S3_SECRET_ACCESS_KEY"), + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=1.0, + read_timeout=2.0, + retries={"max_attempts": 1, "mode": "standard"}, + ), + ) + log.info("storage client initialized") + + +def _bucket() -> str: + return _env("S3_BUCKET", "messenger-attachments") + + +def new_object_key(content_type: str) -> str: + ext = "png" if content_type == "image/png" else "jpg" + return f"attachments/{uuid.uuid4().hex}.{ext}" + + +def put_object(key: str, body: bytes, content_type: str) -> str: + if _client is None: + raise RuntimeError("storage not initialized") + sha256 = hashlib.sha256(body).hexdigest() + _client.put_object( + Bucket=_bucket(), + Key=key, + Body=body, + ContentType=content_type, + Metadata={"sha256": sha256}, + ) + return sha256 + + +def get_object(key: str) -> bytes: + if _client is None: + raise RuntimeError("storage not initialized") + response = _client.get_object(Bucket=_bucket(), Key=key) + return response["Body"].read() + + +def delete_object(key: str) -> None: + if _client is None: + return + try: + _client.delete_object(Bucket=_bucket(), Key=key) + except ClientError as exc: + log.warning("failed to delete object %s: %s", key, exc) + + +def object_exists(key: str) -> bool: + if _client is None: + return False + try: + _client.head_object(Bucket=_bucket(), Key=key) + return True + except ClientError as exc: + if exc.response["Error"]["Code"] in ("404", "NoSuchKey"): + return False + raisev + + +def check_bucket() -> None: + if _client is None: + raise RuntimeError("storage not initialized") + _client.head_bucket(Bucket=_bucket()) \ No newline at end of file diff --git a/src/store.py b/src/store.py new file mode 100755 index 0000000..32fc957 --- /dev/null +++ b/src/store.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import uuid +from datetime import datetime, timezone +from typing import List, Optional, Tuple + +import psycopg +from psycopg.errors import UniqueViolation + +from .db import transaction + +from opentelemetry.propagate import inject as otel_inject +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + + +# ---------- ошибки ---------- + +class ApiError(Exception): + status = 500 + code = "internal_error" + + def __init__(self, message: str = "Internal error") -> None: + super().__init__(message) + self.message = message + + +class BadRequest(ApiError): + status = 400 + code = "invalid_request" + + +class Unauthorized(ApiError): + status = 401 + code = "unauthorized" + + +class NotFound(ApiError): + status = 404 + code = "not_found" + + +class Conflict(ApiError): + status = 409 + code = "conflict" + + +class PayloadTooLarge(ApiError): + status = 413 + code = "payload_too_large" + + +class UnsupportedMediaType(ApiError): + status = 415 + code = "unsupported_media_type" + + +class Unavailable(ApiError): + status = 503 + code = "unavailable" + + +# ---------- утилиты ---------- + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def iso(dt: datetime) -> str: + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _user_dict(row: dict) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } + + +def _chat_dict(row: dict, member_ids: List[str]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": iso(row["created_at"]), + } + + +def _message_dict(row: dict, attachment_ids: List[str]) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso(row["created_at"]), + "attachment_ids": attachment_ids, + } + + +# ---------- курсор ---------- + +def _encode_cursor(scope: str, created_at: str, item_id: str) -> str: + payload = json.dumps( + {"s": scope, "t": created_at, "i": item_id}, separators=(",", ":") + ).encode() + return base64.urlsafe_b64encode(payload).decode().rstrip("=") + + +def _decode_cursor(scope: str, cursor: str) -> Tuple[str, str]: + pad = "=" * (-len(cursor) % 4) + try: + raw = base64.urlsafe_b64decode(cursor + pad) + data = json.loads(raw) + if data.get("s") != scope: + raise ValueError("scope mismatch") + t = data.get("t") + i = data.get("i") + if not isinstance(t, str) or not isinstance(i, str): + raise ValueError("invalid cursor fields") + from datetime import datetime + from uuid import UUID + try: + datetime.fromisoformat(t.replace("Z", "+00:00")) + except (ValueError, TypeError): + raise ValueError("invalid timestamp in cursor") from None + try: + UUID(i) + except (ValueError, TypeError): + raise ValueError("invalid uuid in cursor") from None + return t, i + except Exception as exc: + raise BadRequest("invalid cursor") from exc + +# ---------- Store ---------- + +class Store: + + # ---- users ---- + + def create_user( + self, username: str, display_name: str, password_hash: str + ) -> dict: + uid = str(uuid.uuid4()) + now = _now() + try: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (uid, username, display_name, password_hash, now), + ) + except UniqueViolation: + raise Conflict("username already taken") from None + return { + "id": uid, + "username": username, + "display_name": display_name, + "created_at": iso(now), + } + + def get_user(self, user_id: str) -> Optional[dict]: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + return _user_dict(row) if row else None + + def get_user_by_username(self, username: str) -> Optional[dict]: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE username = %s", (username,)) + row = cur.fetchone() + return row + + # ---- chats ---- + + def create_chat( + self, title: str, member_ids: List[str], creator_id: str + ) -> dict: + if creator_id not in member_ids: + raise BadRequest("creator must be in member_ids") + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT id FROM users WHERE id = ANY(%s)", + (member_ids,), + ) + found = {str(r["id"]) for r in cur.fetchall()} + missing = set(member_ids) - found + if missing: + raise BadRequest("unknown member_id") + + cid = str(uuid.uuid4()) + now = _now() + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (cid, title, now), + ) + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(cid, uid) for uid in member_ids], + ) + return { + "id": cid, + "title": title, + "member_ids": list(member_ids), + "created_at": iso(now), + } + + def get_chat(self, chat_id: str, viewer_id: str) -> Optional[dict]: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT c.* FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s AND m.user_id = %s + """, + (chat_id, viewer_id), + ) + row = cur.fetchone() + if row is None: + return None + cur.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", + (chat_id,), + ) + members = [str(r["user_id"]) for r in cur.fetchall()] + return _chat_dict(row, members) + + def list_chats( + self, viewer_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[dict], Optional[str]]: + scope = "chats:" + viewer_id + params: list = [viewer_id] + where_cursor = "" + if cursor: + t, i = _decode_cursor(scope, cursor) + where_cursor = "AND (c.created_at, c.id) > (%s, %s)" + params.extend([t, i]) + params.append(limit + 1) + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s {where_cursor} + ORDER BY c.created_at, c.id + LIMIT %s + """, + params, + ) + rows = cur.fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + result = [] + for row in page: + cur.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", + (row["id"],), + ) + members = [str(r["user_id"]) for r in cur.fetchall()] + result.append(_chat_dict(row, members)) + + next_cursor = None + if has_more and page: + last = page[-1] + next_cursor = _encode_cursor(scope, iso(last["created_at"]), str(last["id"])) + return result, next_cursor + + # ---- messages ---- + + def add_message( + self, + chat_id: str, + sender_id: str, + text: str, + idempotency_key: str, + attachment_ids: Optional[List[str]] = None, + ) -> tuple[dict, int]: + attachment_ids = attachment_ids or [] + request_hash = hashlib.sha256( + json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + ensure_ascii=False, + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + path = f"/api/v1/chats/{chat_id}/messages" + method = "POST" + + import time as _time + for attempt in range(5): + try: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ) + if cur.fetchone() is None: + raise NotFound() + + cur.execute( + """ + SELECT request_hash, response_status, response_body + FROM idempotency_keys + WHERE user_id = %s AND method = %s AND path = %s AND key = %s + """, + (sender_id, method, path, idempotency_key), + ) + row = cur.fetchone() + if row is not None: + if row["request_hash"] != request_hash: + raise Conflict( + "idempotency key reused with different body" + ) + return row["response_body"], row["response_status"] + + if attachment_ids: + cur.execute( + """ + SELECT id FROM attachments + WHERE id = ANY(%s) AND chat_id = %s + """, + (attachment_ids, chat_id), + ) + found = {str(r["id"]) for r in cur.fetchall()} + missing = set(attachment_ids) - found + if missing: + raise NotFound() + + mid = str(uuid.uuid4()) + now = _now() + cur.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (mid, chat_id, sender_id, text, now), + ) + + if attachment_ids: + cur.executemany( + """ + INSERT INTO message_attachments + (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + [(mid, aid, pos) for pos, aid in enumerate(attachment_ids)], + ) + + message = { + "id": mid, + "chat_id": chat_id, + "sender_id": sender_id, + "text": text, + "created_at": iso(now), + "attachment_ids": list(attachment_ids), + } + + cur.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, path, key, request_hash, + response_status, response_body, created_at) + VALUES (%s, %s, %s, %s, %s, 201, %s, %s) + """, + ( + sender_id, method, path, idempotency_key, + request_hash, + json.dumps(message, ensure_ascii=False), + now, + ), + ) + return message, 201 + except UniqueViolation: + _time.sleep(0.05) + continue + + raise Conflict("idempotency key in progress, please retry") + + def list_messages( + self, + chat_id: str, + viewer_id: str, + limit: int, + cursor: Optional[str], + ) -> Tuple[List[dict], Optional[str]]: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, viewer_id), + ) + if cur.fetchone() is None: + raise NotFound() + + scope = "chat:" + chat_id + params: list = [chat_id] + where_cursor = "" + if cursor: + t, i = _decode_cursor(scope, cursor) + where_cursor = "AND (created_at, id) > (%s, %s)" + params.extend([t, i]) + params.append(limit + 1) + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s {where_cursor} + ORDER BY created_at, id + LIMIT %s + """, + params, + ) + rows = cur.fetchall() + + message_ids = [str(r["id"]) for r in rows] + attachments_by_message: dict = {} + if message_ids: + cur.execute( + """ + SELECT message_id, attachment_id + FROM message_attachments + WHERE message_id = ANY(%s) + ORDER BY position + """, + (message_ids,), + ) + for r in cur.fetchall(): + attachments_by_message.setdefault( + str(r["message_id"]), [] + ).append(str(r["attachment_id"])) + + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for r in page: + result.append(_message_dict( + r, attachments_by_message.get(str(r["id"]), []) + )) + + next_cursor = None + if has_more and page: + last = page[-1] + next_cursor = _encode_cursor(scope, iso(last["created_at"]), str(last["id"])) + return result, next_cursor + + # ---- attachments ---- + + def create_attachment( + self, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + s3_key: str, + staging_key: Optional[str] = None, + status: str = "queued", + ) -> dict: + aid = str(uuid.uuid4()) + now = _now() + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, owner_id), + ) + if cur.fetchone() is None: + raise NotFound() + cur.execute( + """ + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, s3_key, staging_key, + status, error_code, attempt, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NULL, 0, %s, %s) + """, + (aid, chat_id, owner_id, filename, content_type, + size_bytes, sha256, s3_key, staging_key, + status, now, now), + ) + return { + "id": aid, + "chat_id": chat_id, + "owner_id": owner_id, + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "sha256": sha256, + "status": status, + "variants": [], + "error_code": None, + "created_at": iso(now), + "updated_at": iso(now), + } + + def create_job(self, attachment_id: str, pipeline_version: int = 1) -> dict: + jid = str(uuid.uuid4()) + now = _now() + carrier: dict = {} + otel_inject(carrier) + traceparent = carrier.get("traceparent") + tracestate = carrier.get("tracestate") + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO jobs + (job_id, attachment_id, pipeline_version, + attempt, status, created_at, traceparent, tracestate) + VALUES (%s, %s, %s, 1, 'pending', %s, %s, %s) + """, + (jid, attachment_id, pipeline_version, now, traceparent, tracestate), + ) + return { + "schema_version": 1, + "job_id": jid, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": pipeline_version, + "attempt": 1, + "created_at": iso(now), + } + + def mark_job_published(self, job_id: str) -> None: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE jobs SET status='published', published_at=NOW() " + "WHERE job_id=%s", + (job_id,), + ) + + def get_attachment( + self, attachment_id: str, viewer_id: str + ) -> Optional[dict]: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id + WHERE a.id = %s AND m.user_id = %s + """, + (attachment_id, viewer_id), + ) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "s3_key": row["s3_key"], + "status": row["status"], + "variants": row["variants"] if row.get("variants") else [], + "error_code": row["error_code"], + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +store = Store() \ No newline at end of file diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..073a0de --- /dev/null +++ b/src/worker.py @@ -0,0 +1,377 @@ +from __future__ import annotations + +import io +import logging +import os +import time +import random + +from PIL import Image, ImageOps + +from . import broker, storage +from .db import init_pool, close_pool, transaction +from .store import store, iso + +from opentelemetry import trace, metrics +from opentelemetry.trace import SpanKind + +from . import otel as otel_module +from .logging_config import configure_logging +from prometheus_client import ( + Counter as _PromCounter, + Histogram as _PromHistogram, + Gauge as _PromGauge, + CollectorRegistry as _PromRegistry, + generate_latest as _prom_generate, + CONTENT_TYPE_LATEST as _PROM_CONTENT_TYPE, +) + +configure_logging("messenger-worker") +log = logging.getLogger("messenger.worker") + +THUMB_SIZE = (256, 256) +MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +_PROM_REGISTRY = _PromRegistry() +_JOBS_PROCESSED = _PromCounter( + "jobs_processed_total", + "Jobs processed by outcome", + ["job_type", "outcome"], + registry=_PROM_REGISTRY, +) +_JOB_DURATION = _PromHistogram( + "job_processing_duration_seconds", + "Job processing duration in seconds", + ["job_type", "outcome"], + buckets=(0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0), + registry=_PROM_REGISTRY, +) +_WORKER_IN_FLIGHT = _PromGauge( + "worker_in_flight", + "Jobs currently being processed by this worker", + registry=_PROM_REGISTRY, +) +_WORKER_CAPACITY = _PromGauge( + "worker_capacity", + "Worker prefetch capacity (WORKER_CONCURRENCY)", + registry=_PROM_REGISTRY, +) + +_tracer = trace.get_tracer("messenger.worker") + + +def _load_original(staging_key: str) -> bytes: + return storage.get_object(staging_key) + + +def _make_thumbnail(data: bytes) -> tuple[bytes, int, int]: + img = Image.open(io.BytesIO(data)) + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA", "P"): + background = Image.new("RGB", img.size, (255, 255, 255)) + rgba = img.convert("RGBA") + background.paste(rgba, mask=rgba.split()[-1]) + img = background + else: + img = img.convert("RGB") + img.thumbnail(THUMB_SIZE, Image.LANCZOS) + w, h = img.size + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85, optimize=True) + return buf.getvalue(), w, h + +def _make_crop(data: bytes, size: int = 128) -> tuple[bytes, int, int]: + img = Image.open(io.BytesIO(data)) + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA", "P"): + background = Image.new("RGB", img.size, (255, 255, 255)) + rgba = img.convert("RGBA") + background.paste(rgba, mask=rgba.split()[-1]) + img = background + else: + img = img.convert("RGB") + w, h = img.size + side = min(w, h) + left = (w - side) // 2 + top = (h - side) // 2 + img = img.crop((left, top, left + side, top + side)) + img = img.resize((size, size), Image.LANCZOS) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85, optimize=True) + return buf.getvalue(), size, size + + +def _make_optimized(data: bytes, max_side: int = 1280) -> tuple[bytes, int, int]: + img = Image.open(io.BytesIO(data)) + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA", "P"): + background = Image.new("RGB", img.size, (255, 255, 255)) + rgba = img.convert("RGBA") + background.paste(rgba, mask=rgba.split()[-1]) + img = background + else: + img = img.convert("RGB") + img.thumbnail((max_side, max_side), Image.LANCZOS) + w, h = img.size + buf = io.BytesIO() + img.save(buf, format="WEBP", quality=80, method=4) + return buf.getvalue(), w, h + +def _handle(job: dict, parent_ctx=None) -> None: + attachment_id = job["attachment_id"] + job_type = job.get("type", "attachment.process.v1") + attempt = int(job.get("attempt", 1)) + + with _tracer.start_as_current_span( + "job.process", + context=parent_ctx, + kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": "attachment.process.v1", + "messaging.operation": "process", + "messaging.message.id": job["job_id"], + "job.type": job_type, + "job.attempt": attempt, + "job.attachment_id": attachment_id, + }, + ) as span: + _WORKER_IN_FLIGHT.inc() + import time as _t + _start = _t.monotonic() + outcome = "success" + try: + _handle_inner(job) + except Exception as exc: + outcome = "failure" + span.record_exception(exc) + span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc))) + raise + finally: + elapsed = _t.monotonic() - _start + _JOB_DURATION.labels(job_type=job_type, outcome=outcome).observe(elapsed) + _JOBS_PROCESSED.labels(job_type=job_type, outcome=outcome).inc() + _WORKER_IN_FLIGHT.dec() + + +def _handle_inner(job: dict) -> None: + attachment_id = job["attachment_id"] + attempt = int(job.get("attempt", 1)) + job_id = job["job_id"] + log.info("processing attachment %s attempt %d (job %s)", attachment_id, attempt, job_id) + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE jobs + SET status = 'processing', + lease_until = NOW() + INTERVAL '60 seconds' + WHERE job_id = %s AND status IN ('published', 'pending') + RETURNING job_id + """, + (job_id,), + ) + acquired = cur.fetchone() + if acquired is None: + log.info("job %s already acquired or done, skip", job_id) + return + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE attachments SET status='processing', updated_at=NOW() " + "WHERE id=%s AND status='queued'", + (attachment_id,), + ) + cur.execute( + "SELECT id, chat_id, owner_id, filename, content_type, " + "size_bytes, sha256, s3_key, staging_key, status " + "FROM attachments WHERE id = %s", + (attachment_id,), + ) + row = cur.fetchone() + if row is None: + log.error("attachment %s not found, skip", attachment_id) + return + if row["status"] == "ready": + log.info("attachment %s already ready, no-op", attachment_id) + return + + staging_key = row.get("staging_key") or row["s3_key"] + + data = _load_original(staging_key) + + thumb_bytes, tw, th = _make_thumbnail(data) + crop_bytes, cw, ch = _make_crop(data, 128) + opt_bytes, ow, oh = _make_optimized(data, 1280) + + ext = row["content_type"].split("/")[-1] + final_key = f"originals/{attachment_id}.{ext}" + if staging_key != final_key: + storage.put_object(final_key, data, row["content_type"]) + + thumb_key = f"thumbnails/{attachment_id}.jpg" + thumb_sha = storage.put_object(thumb_key, thumb_bytes, "image/jpeg") + + crop_key = f"crops/{attachment_id}.jpg" + crop_sha = storage.put_object(crop_key, crop_bytes, "image/jpeg") + + opt_key = f"optimized/{attachment_id}.webp" + opt_sha = storage.put_object(opt_key, opt_bytes, "image/webp") + + import json + variants = [ + { + "name": "thumbnail", + "content_type": "image/jpeg", + "size_bytes": len(thumb_bytes), + "width": tw, + "height": th, + "sha256": thumb_sha, + }, + { + "name": "crop", + "content_type": "image/jpeg", + "size_bytes": len(crop_bytes), + "width": cw, + "height": ch, + "sha256": crop_sha, + }, + { + "name": "optimized", + "content_type": "image/webp", + "size_bytes": len(opt_bytes), + "width": ow, + "height": oh, + "sha256": opt_sha, + }, + ] + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE attachments + SET status = 'ready', + s3_key = %s, + variants = %s::jsonb, + error_code = NULL, + updated_at = NOW() + WHERE id = %s + """, + (final_key, json.dumps(variants), attachment_id), + ) + cur.execute( + "UPDATE jobs SET status='published', lease_until=NULL, published_at=NOW() " + "WHERE job_id=%s", + (job_id,), + ) + + if staging_key and staging_key != final_key: + try: + storage.delete_object(staging_key) + except Exception as exc: + log.warning("staging cleanup failed: %s", exc) + + log.info( + "attachment %s ready: thumbnail %dx%d, crop %dx%d, optimized %dx%d", + attachment_id, tw, th, cw, ch, ow, oh, + ) + + +PERMANENT_ERRORS = ("invalid_image", "unsupported_media_type", "payload_too_large") + + +def _is_permanent(exc: Exception) -> bool: + name = type(exc).__name__.lower() + msg = str(exc).lower() + return any(p in name or p in msg for p in PERMANENT_ERRORS) + + +def _handler_with_retry(job: dict, parent_ctx=None) -> None: + attempt = int(job.get("attempt", 1)) + try: + _handle(job, parent_ctx=parent_ctx) + except Exception as exc: + log.exception("job %s attempt %d failed: %s", job.get("job_id"), attempt, exc) + permanent = _is_permanent(exc) + if permanent or attempt >= MAX_ATTEMPTS: + try: + broker.publish_dlq(job, str(exc)) + except Exception as dlq_exc: + log.warning("DLQ publish failed: %s", dlq_exc) + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE attachments SET status='failed', " + "error_code=%s, updated_at=NOW() WHERE id=%s", + ("invalid_image" if permanent else "processing_failed", + job["attachment_id"]), + ) + cur.execute( + "UPDATE jobs SET status='failed', lease_until=NULL " + "WHERE job_id=%s", + (job["job_id"],), + ) + log.error( + "attachment %s failed (permanent=%s) after %d attempts", + job["attachment_id"], permanent, attempt, + ) + return + delay = min(60, 2 ** attempt) + random.uniform(0, 1) + log.info("retry in %.1fs (attempt %d -> %d)", delay, attempt, attempt + 1) + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE jobs SET status='pending', lease_until=NULL, attempt=%s " + "WHERE job_id=%s", + (attempt + 1, job["job_id"]), + ) + + +def main() -> None: + otel_module.setup_otel("messenger-worker") + otel_module.instrument_libraries(psycopg=True, botocore=True, redis=False) + + init_pool() + storage.init_storage() + _WORKER_CAPACITY.set(int(os.environ.get("WORKER_CONCURRENCY", "2"))) + + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + class MetricsHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/metrics": + data = _prom_generate(_PROM_REGISTRY) + self.send_response(200) + self.send_header("Content-Type", _PROM_CONTENT_TYPE) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + elif self.path == "/health/live": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + else: + self.send_response(404) + self.end_headers() + + def log_message(self, *args): + pass + + metrics_port = int(os.environ.get("WORKER_METRICS_PORT", "9100")) + httpd = ThreadingHTTPServer(("0.0.0.0", metrics_port), MetricsHandler) + threading.Thread(target=httpd.serve_forever, daemon=True, name="metrics-http").start() + log.info("worker metrics on :%d/metrics", metrics_port) + + log.info("worker starting, max_attempts=%d", MAX_ATTEMPTS) + try: + broker.consume(_handler_with_retry, prefetch=int(os.environ.get("WORKER_CONCURRENCY", "2"))) + finally: + otel_module.shutdown_otel() + close_pool() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/seed_messages.py b/student_tests/seed_messages.py new file mode 100644 index 0000000..401e304 --- /dev/null +++ b/student_tests/seed_messages.py @@ -0,0 +1,89 @@ +"""Генерация тестовых сообщений для EXPLAIN. + +Создаёт пользователя, чат и N сообщений. N по умолчанию 10 000. +Запуск: + python3 seed_messages.py --base-url http://localhost:8080 --count 10000 +""" +from __future__ import annotations + +import argparse +import base64 +import json +import time +import urllib.error +import urllib.request +import uuid + + +def call(base: str, method: str, path: str, headers: dict | None = None, body: dict | None = None) -> tuple[int, dict | None]: + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + req = urllib.request.Request(base + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=30) + raw = resp.read() + return resp.code, json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) if raw else None + except Exception: + return e.code, None + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--count", type=int, default=10000) + p.add_argument("--batch", type=int, default=500, help="Сообщений между progress-логами") + args = p.parse_args() + base = args.base_url.rstrip("/") + + username = "seed_" + uuid.uuid4().hex[:12] + password = "seed-pass-1234567890" + + # Регистрация + code, user = call(base, "POST", "/api/v1/users", body={ + "username": username, "display_name": "Seed User", "password": password, + }) + assert code == 201, f"register failed: {code} {user}" + + # Логин + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session = call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code} {session}" + token = session["token"] + user_id = session["user"]["id"] + + # Чат + code, chat = call(base, "POST", "/api/v1/chats", headers={"Authorization": "Bearer " + token}, body={ + "title": "Seed chat", "member_ids": [user_id], + }) + assert code == 201, f"chat failed: {code} {chat}" + chat_id = chat["id"] + print(f"chat_id: {chat_id}") + print(f"seeding {args.count} messages...") + + started = time.monotonic() + for i in range(args.count): + code, msg = call( + base, "POST", f"/api/v1/chats/{chat_id}/messages", + headers={"Authorization": "Bearer " + token}, + body={"text": f"seed-{i:06d}-{uuid.uuid4().hex[:8]}"}, + ) + assert code == 201, f"message {i} failed: {code} {msg}" + if (i + 1) % args.batch == 0: + elapsed = time.monotonic() - started + rps = (i + 1) / elapsed + print(f" {i + 1}/{args.count} ({rps:.0f} msg/s)") + + elapsed = time.monotonic() - started + print(f"done: {args.count} messages in {elapsed:.1f}s ({args.count / elapsed:.0f} msg/s)") + print(f"chat_id: {chat_id}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_attachments_negative.py b/student_tests/test_attachments_negative.py new file mode 100644 index 0000000..d6e1959 --- /dev/null +++ b/student_tests/test_attachments_negative.py @@ -0,0 +1,292 @@ +"""Негативные тесты для вложений (lab4 grade 4). + +Запуск: + python3 test_attachments_negative.py --base-url http://localhost:8080 +""" +from __future__ import annotations + +import base64 +import io +import json +import sys +import urllib.error +import urllib.request +import uuid +import time + +BASE = ( + sys.argv[sys.argv.index("--base-url") + 1] + if "--base-url" in sys.argv + else "http://localhost:8080" +) + + +def call(method, path, headers=None, body=None, raw=None): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + data = raw + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=30) + payload = resp.read() + return resp.status, json.loads(payload) if payload else None, dict(resp.headers) + except urllib.error.HTTPError as e: + payload = e.read() + try: + return e.code, json.loads(payload) if payload else None, dict(e.headers) + except Exception: + return e.code, None, dict(e.headers) + + +def make_user(label): + username = f"att_{label}_{uuid.uuid4().hex[:10]}" + password = "negative-pass-1234567890" + code, user, _ = call("POST", "/api/v1/users", body={ + "username": username, "display_name": f"Att {label}", "password": password, + }) + assert code == 201, f"register {label} failed: {code} {user}" + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session, _ = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login {label} failed: {code}" + return { + "id": user["id"], + "token": session["token"], + } + + +def make_chat(owner, member_ids=None): + if member_ids is None: + member_ids = [owner["id"]] + code, chat, _ = call("POST", "/api/v1/chats", + headers={"Authorization": "Bearer " + owner["token"]}, + body={"title": "Negative", "member_ids": member_ids}) + assert code == 201, f"chat failed: {code} {chat}" + return chat["id"] + + +def upload(token, chat_id, filename, data, content_type): + return call( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers={"Authorization": "Bearer " + token, "Content-Type": content_type}, + raw=data, + ) + + +def test_decompression_bomb(): + """PNG 8200x5000 (41M пикселей) — маленький по размеру, но огромный по пикселям.""" + from PIL import Image + import io as _io + alice = make_user("bomb") + chat = make_chat(alice) + + img = Image.new("RGB", (8200, 5000), color=(255, 0, 0)) + buf = _io.BytesIO() + img.save(buf, format="PNG", compress_level=9) + data = buf.getvalue() + assert len(data) < 500 * 1024, f"expected small compressed, got {len(data)}" + + started = time.monotonic() + code, _, _ = upload(alice["token"], chat, "bomb.png", data, "image/png") + elapsed = time.monotonic() - started + + assert code == 413, f"expected 413, got {code}" + assert elapsed < 2.0, f"expected fast rejection (<2s), got {elapsed:.2f}s" + print(f"PASS: decompression bomb -> 413 ({len(data)} bytes, {elapsed:.2f}s)") + + +def test_invalid_cursor(): + """Cursor с невалидным timestamp или UUID -> 400.""" + import base64 as _b64 + alice = make_user("cursor") + chat = make_chat(alice) + chat_id = chat + + # Невалидный timestamp + bad_t = _b64.urlsafe_b64encode( + json.dumps({ + "s": f"chat:{chat_id}", + "t": "not-a-time", + "i": "00000000-0000-4000-8000-000000000000", + }).encode() + ).decode().rstrip("=") + code, _, _ = call( + "GET", f"/api/v1/chats/{chat_id}/messages?cursor={bad_t}", + headers={"Authorization": "Bearer " + alice["token"]}, + ) + assert code == 400, f"invalid timestamp: expected 400, got {code}" + + # Невалидный UUID + bad_i = _b64.urlsafe_b64encode( + json.dumps({ + "s": f"chat:{chat_id}", + "t": "2026-01-01T00:00:00.000000Z", + "i": "not-a-uuid", + }).encode() + ).decode().rstrip("=") + code, _, _ = call( + "GET", f"/api/v1/chats/{chat_id}/messages?cursor={bad_i}", + headers={"Authorization": "Bearer " + alice["token"]}, + ) + assert code == 400, f"invalid uuid: expected 400, got {code}" + + print("PASS: invalid cursor -> 400 (timestamp and uuid)") + + +def test_filename_c1(): + """Filename с C1 control (U+0085) -> 400.""" + from pathlib import Path + fixture = Path(__file__).resolve().parent.parent / "tests/fixtures/sample.png" + png = fixture.read_bytes() + alice = make_user("c1") + chat = make_chat(alice) + + # filename = "c1\u0085.png" — C1 control U+0085 + code, _, _ = call( + "POST", + f"/api/v1/chats/{chat}/attachments?filename=c1%C2%85.png", + headers={ + "Authorization": "Bearer " + alice["token"], + "Content-Type": "image/png", + }, + raw=png, + ) + assert code == 400, f"expected 400 for C1 filename, got {code}" + print("PASS: C1 filename -> 400") + + +def test_s3_unavailable(): + """S3 down: readiness 503, upload 503.""" + import subprocess + from pathlib import Path + + alice = make_user("s3down") + chat = make_chat(alice) + + # 1. Baseline: readiness 200 + code, _, _ = call("GET", "/health/ready") + assert code == 200, f"readiness should be 200 before stop, got {code}" + + # 2. Останавливаем S3 + try: + subprocess.run( + ["docker", "compose", "--env-file", ".env", "stop", "s3"], + check=True, capture_output=True, text=True, + cwd=str(Path(__file__).resolve().parent.parent), + timeout=30, + ) + except subprocess.CalledProcessError as e: + raise SystemExit(f"failed to stop s3: {e.stderr}") + + import time as _t + _t.sleep(3) + + try: + # 3. Readiness должен быть 503 + code, body, _ = call("GET", "/health/ready") + assert code == 503, f"readiness should be 503 with S3 down, got {code}" + assert body and body["error"]["code"] == "unavailable", \ + f"expected unavailable, got {body}" + + # 4. Upload должен быть 503 + fixture = Path(__file__).resolve().parent.parent / "tests/fixtures/sample.png" + png = fixture.read_bytes() + code, body, _ = upload(alice["token"], chat, "s3down.png", png, "image/png") + assert code == 503, f"upload should be 503 with S3 down, got {code}" + assert body and body["error"]["code"] == "unavailable", \ + f"expected unavailable, got {body}" + + print("PASS: S3 down -> readiness 503, upload 503") + finally: + # 5. Возвращаем S3 + subprocess.run( + ["docker", "compose", "--env-file", ".env", "start", "s3"], + check=False, capture_output=True, text=True, + cwd=str(Path(__file__).resolve().parent.parent), + timeout=30, + ) + # Ждём готовности + deadline = _t.monotonic() + 30 + while _t.monotonic() < deadline: + code, _, _ = call("GET", "/health/ready") + if code == 200: + break + _t.sleep(1) + + +def main(): + alice = make_user("alice") + bob = make_user("bob") + eve = make_user("eve") + chat_ab = make_chat(alice, [alice["id"], bob["id"]]) + chat_alice = make_chat(alice) + + # 1. Oversize: > 10 MiB + big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10 * 1024 * 1024 + 1) + code, _, _ = upload(alice["token"], chat_ab, "big.png", big, "image/png") + assert code == 413, f"oversize: expected 413, got {code}" + print("PASS: oversize -> 413") + + # 2. Corrupted: валидная сигнатура, битые данные + corrupted = b"\x89PNG\r\n\x1a\n" + b"\xff" * 100 + code, _, _ = upload(alice["token"], chat_ab, "corrupted.png", corrupted, "image/png") + assert code == 415, f"corrupted: expected 415, got {code}" + print("PASS: corrupted -> 415") + + # 3. MIME-spoofing: image/png, но текст + spoofed = b"this is definitely not an image" + code, _, _ = upload(alice["token"], chat_ab, "spoof.png", spoofed, "image/png") + assert code == 415, f"mime spoof: expected 415, got {code}" + print("PASS: mime spoofing -> 415") + + # 4. Чужой attachment_id: eve скачивает вложение из chat_ab + # Сначала загрузим настоящий PNG (тестовый fixture) + from pathlib import Path + fixture = Path(__file__).resolve().parent.parent / "tests/fixtures/sample.png" + if not fixture.exists(): + raise SystemExit(f"fixture not found: {fixture}") + png = fixture.read_bytes() + code, att, _ = upload(alice["token"], chat_ab, "sample.png", png, "image/png") + assert code == 201, f"upload fixture failed: {code} {att}" + att_id = att["id"] + + code, _, _ = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + eve["token"]}) + assert code == 404, f"outsider metadata: expected 404, got {code}" + code, _, _ = call("GET", f"/api/v1/attachments/{att_id}/content", + headers={"Authorization": "Bearer " + eve["token"]}) + assert code == 404, f"outsider content: expected 404, got {code}" + print("PASS: outsider attachment -> 404") + + # 5. Вложение из другого чата + # Загружаем в chat_alice, прикрепляем к сообщению в chat_ab + code, att2, _ = upload(alice["token"], chat_alice, "other.png", png, "image/png") + assert code == 201, f"upload to chat_alice failed: {code} {att2}" + other_id = att2["id"] + + code, _, _ = call( + "POST", f"/api/v1/chats/{chat_ab}/messages", + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + body={"text": "cross-chat", "attachment_ids": [other_id]}, + ) + assert code == 404, f"cross-chat attachment: expected 404, got {code}" + print("PASS: cross-chat attachment -> 404") + + print("\nNew tests from reviewer feedback") + test_decompression_bomb() + test_invalid_cursor() + test_filename_c1() + + print("\nS3 unavailable") + test_s3_unavailable() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_cache_failure.py b/student_tests/test_cache_failure.py new file mode 100755 index 0000000..00ab135 --- /dev/null +++ b/student_tests/test_cache_failure.py @@ -0,0 +1,100 @@ +"""Проверка отказа Valkey и корректности logout при недоступном кэше.""" +from __future__ import annotations +import base64, json, subprocess, sys, time, urllib.error, urllib.request, uuid + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + +def call(method, path, headers=None, body=None): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + r = urllib.request.urlopen(req, timeout=10) + raw = r.read() + return r.code, json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) if raw else None + except Exception: + return e.code, None + +def stop_cache(): + subprocess.run(COMPOSE + ["stop", "cache"], check=True) + +def start_cache(): + subprocess.run(COMPOSE + ["start", "cache"], check=True) + +def restart_api(): + subprocess.run(COMPOSE + ["restart", "api"], check=True) + +def wait_ready(deadline=60): + end = time.monotonic() + deadline + while time.monotonic() < end: + try: + code, _ = call("GET", "/health/ready") + if code == 200: + return + except Exception: + pass + time.sleep(1) + raise SystemExit("API did not become ready") + +def main(): + username = "cf_" + uuid.uuid4().hex[:10] + password = "cache-fail-pass-1234567890" + code, _ = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "Cache Fail", "password": password, + }) + assert code == 201, code + + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, code + token = session["token"] + + # Токен работает + code, _ = call("GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}) + assert code == 200, code + + # Останавливаем Valkey + stop_cache() + time.sleep(2) + + # API должно работать в degraded mode + code, _ = call("GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}) + assert code == 200, f"degraded mode failed: {code}" + print("PASS: API works with Valkey down (degraded mode)") + + # Logout при недоступном Valkey должен вернуть 204 + code, _ = call("DELETE", "/api/v1/auth/sessions/current", + headers={"Authorization": "Bearer " + token}) + assert code == 204, f"logout with cache down returned {code}" + print("PASS: logout with cache down returned 204") + + # Токен немедленно недействителен (проверка через БД) + code, _ = call("GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}) + assert code == 401, f"revoked token still works with cache down: {code}" + print("PASS: revoked token rejected while cache down") + + # Поднимаем Valkey, рестартуем API — критический регресс-тест + start_cache() + time.sleep(2) + restart_api() + wait_ready() + + code, _ = call("GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}) + assert code == 401, f"REGRESSION: revoked token resurrected after cache restart: {code}" + print("PASS: revoked token still 401 after cache restart (regression fixed)") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_concurrent_uploads.py b/student_tests/test_concurrent_uploads.py new file mode 100644 index 0000000..701f9d6 --- /dev/null +++ b/student_tests/test_concurrent_uploads.py @@ -0,0 +1,179 @@ +"""10 одновременных загрузок 10 MiB: все 202, все ready, RAM API. + +Запуск: + python3 student_tests/test_concurrent_uploads.py --base-url http://localhost:8080 +""" +from __future__ import annotations + +import base64 +import io +import json +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + +BASE = ( + sys.argv[sys.argv.index("--base-url") + 1] + if "--base-url" in sys.argv + else "http://localhost:8080" +) +CONCURRENCY = 10 +SIZE_MB = 10 +API_CONTAINER = "mtusi2026lab5-api-1" + + +def call(method, path, headers=None, body=None, raw=None, timeout=120): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + data = raw + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=timeout) + payload = resp.read() + return resp.status, json.loads(payload) if payload else None + except urllib.error.HTTPError as e: + payload = e.read() + try: + return e.code, json.loads(payload) if payload else None + except Exception: + return e.code, None + + +def make_user(label): + username = f"conc_{label}_{uuid.uuid4().hex[:10]}" + password = "conc-pass-1234567890" + code, user = call("POST", "/api/v1/users", body={ + "username": username, "display_name": f"Conc {label}", "password": password, + }) + assert code == 201, f"register failed: {code} {user}" + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code}" + code, chat = call("POST", "/api/v1/chats", + headers={"Authorization": "Bearer " + session["token"]}, + body={"title": "Conc", "member_ids": [user["id"]]}) + assert code == 201, f"chat failed: {code}" + return {"token": session["token"], "chat_id": chat["id"]} + + +def measure_ram_mb(container): + try: + result = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container], + capture_output=True, text=True, timeout=5, + ) + if result.returncode != 0: + return None + mem = result.stdout.strip().split("/")[0].strip() + if "MiB" in mem: + return float(mem.replace("MiB", "").strip()) + if "GiB" in mem: + return float(mem.replace("GiB", "").strip()) * 1024 + if "KiB" in mem: + return float(mem.replace("KiB", "").strip()) / 1024 + return None + except Exception: + return None + + +def make_png_of_size(size_bytes): + from PIL import Image + side = int((size_bytes / 3) ** 0.5) + img = Image.new("RGB", (side, side), color=(200, 100, 50)) + buf = io.BytesIO() + img.save(buf, format="PNG", compress_level=0) + return buf.getvalue() + + +def main(): + print(f"=== concurrent uploads: {CONCURRENCY} × {SIZE_MB} MiB ===") + user = make_user("user") + png = make_png_of_size(SIZE_MB * 1024 * 1024) + print(f"fixture size: {len(png)} bytes ({len(png) / 1024 / 1024:.2f} MiB)") + + baseline = measure_ram_mb(API_CONTAINER) + print(f"baseline RAM: {baseline:.1f} MiB" if baseline else "baseline RAM: unknown") + + peak = baseline or 0.0 + stop = threading.Event() + + def monitor(): + nonlocal peak + while not stop.is_set(): + ram = measure_ram_mb(API_CONTAINER) + if ram is not None: + peak = max(peak, ram) + time.sleep(0.2) + + t = threading.Thread(target=monitor, daemon=True) + t.start() + + results = [] + lock = threading.Lock() + barrier = threading.Barrier(CONCURRENCY) + + def worker(i): + barrier.wait() + code, body = call( + "POST", + f"/api/v1/chats/{user['chat_id']}/attachments?filename=conc-{i}.png", + headers={ + "Authorization": "Bearer " + user["token"], + "Content-Type": "image/png", + }, + raw=png, + ) + with lock: + results.append((i, code, body)) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + list(pool.map(worker, range(CONCURRENCY))) + elapsed = time.monotonic() - started + stop.set() + t.join(timeout=2) + + codes = {} + for _, code, _ in results: + codes[code] = codes.get(code, 0) + 1 + + print(f"\nUpload results:") + print(f" duration: {elapsed:.2f}s") + print(f" status codes: {codes}") + print(f" peak RAM: {peak:.1f} MiB (baseline {baseline:.1f} MiB)") + assert all(c == 202 for _, c, _ in results), f"not all 202: {codes}" + + # Ждём все ready + att_ids = [body["id"] for _, _, body in results] + deadline = time.monotonic() + 120 + ready_count = 0 + while time.monotonic() < deadline: + ready_count = 0 + for att_id in att_ids: + code, att = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + user["token"]}) + if code == 200 and att["status"] == "ready": + ready_count += 1 + elif code == 200 and att["status"] == "failed": + raise AssertionError(f"att {att_id} failed: {att.get('error_code')}") + if ready_count == len(att_ids): + break + time.sleep(1) + + assert ready_count == len(att_ids), f"only {ready_count}/{len(att_ids)} ready" + print(f" all {len(att_ids)} ready in {time.monotonic() - (deadline - 120):.1f}s") + print(f"\nPASS: {CONCURRENCY} concurrent uploads 202→ready, peak RAM {peak:.1f} MiB") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_db_failure.py b/student_tests/test_db_failure.py new file mode 100755 index 0000000..4dfe26d --- /dev/null +++ b/student_tests/test_db_failure.py @@ -0,0 +1,127 @@ +"""Проверка отказа PostgreSQL: таймаут запроса и восстановление.""" +from __future__ import annotations +import base64, json, subprocess, sys, time, urllib.error, urllib.request, uuid + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def call(method, path, headers=None, body=None, timeout=10): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + r = urllib.request.urlopen(req, timeout=timeout) + raw = r.read() + return r.code, json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) if raw else None + except Exception: + return e.code, None + + +def stop_db(): + """Останавливает контейнер Postgres. TCP-соединения рвутся, psycopg + получает ошибку мгновенно — это честный сценарий 'БД упала'.""" + subprocess.run(COMPOSE + ["stop", "postgres"], check=True) + + +def start_db(): + """Поднимает Postgres. Не падает, если уже запущен.""" + subprocess.run(COMPOSE + ["start", "postgres"], + capture_output=True, check=False) + time.sleep(3) + + +def restart_api(): + """Пересоздаёт API, чтобы пул соединений не держал мёртвые сессии.""" + subprocess.run(COMPOSE + ["restart", "api"], + capture_output=True, check=False) + + +def wait_ready(deadline=60): + end = time.monotonic() + deadline + while time.monotonic() < end: + try: + code, _ = call("GET", "/health/ready", timeout=5) + if code == 200: + return + except Exception: + pass + time.sleep(1) + raise SystemExit("API did not become ready") + + +def ensure_clean_state(): + """Возвращает стек в рабочее состояние, если предыдущий запуск оставил + Postgres остановленным или пул API в плохом состоянии.""" + subprocess.run(COMPOSE + ["start", "postgres"], + capture_output=True, check=False) + time.sleep(3) + # API перезапускаем, потому что после stop Postgres пул мог забиться + # мёртвыми соединениями, и первый запрос упадёт по PoolTimeout. + restart_api() + wait_ready() + + +def main(): + ensure_clean_state() + + # Регистрация и логин — до всякого stop, чтобы тест не падал на подготовке + username = "dbf_" + uuid.uuid4().hex[:10] + password = "db-fail-pass-1234567890" + + code, _ = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "DB Fail", "password": password, + }) + assert code == 201, f"failed to create user before stop: {code}" + + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"failed to login before stop: {code}" + token = session["token"] + + stop_db() + try: + time.sleep(0.5) + + # Запрос к БД должен упасть быстро (TCP порван, psycopg сразу + # получает ошибку соединения). Проверяем < 8 секунд. + start = time.monotonic() + try: + code, _ = call("GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}, timeout=15) + except Exception as exc: + elapsed = time.monotonic() - start + raise SystemExit(f"request hung: {elapsed:.1f}s, {type(exc).__name__}") + elapsed = time.monotonic() - start + assert elapsed < 8, f"request took {elapsed:.1f}s (>8s), DB down not handled fast enough" + assert code in (500, 503), f"expected 500/503 during DB down, got {code}" + print(f"PASS: DB down returned {code} in {elapsed:.1f}s (<8s)") + + # health/ready должен вернуть 503, пока БД недоступна + code, _ = call("GET", "/health/ready", timeout=15) + assert code == 503, f"expected 503 from ready, got {code}" + print("PASS: /health/ready returned 503 during DB down") + finally: + # Гарантированно поднимаем Postgres даже при падении выше. + start_db() + # Перезапускаем API, чтобы пул пересоздал соединения с вернувшейся БД. + restart_api() + wait_ready() + + # Проверяем, что данные и сессия сохранены после восстановления. + code, _ = call("GET", "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}) + assert code == 200, f"session lost after DB recovery: {code}" + print("PASS: session and data preserved after DB recovery") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_expiry.py b/student_tests/test_expiry.py new file mode 100644 index 0000000..68d553c --- /dev/null +++ b/student_tests/test_expiry.py @@ -0,0 +1,72 @@ +"""Тест expiry: сессия истекает через SESSION_TTL_SECONDS. + +Запуск: + SESSION_TTL_SECONDS=2 docker compose up -d --build api + python3 test_expiry.py --base-url http://localhost:8080 + +Тест создаёт пользователя, логинится, ждёт TTL+1 секунду, проверяет 401. +""" +from __future__ import annotations + +import base64 +import json +import sys +import time +import urllib.error +import urllib.request +import uuid + +BASE = sys.argv[sys.argv.index("--base-url") + 1] if "--base-url" in sys.argv else "http://localhost:8080" + + +def call(method: str, path: str, headers: dict | None = None, body: dict | None = None) -> tuple[int, dict | None]: + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=10) + raw = resp.read() + return resp.code, json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) if raw else None + except Exception: + return e.code, None + + +def main() -> None: + username = "exp_" + uuid.uuid4().hex[:12] + password = "expiry-pass-1234567890" + + # Регистрация + code, user = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "Expiry Test", "password": password, + }) + assert code == 201, f"register failed: {code} {user}" + + # Логин + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code} {session}" + token = session["token"] + print(f"session expires_at: {session['expires_at']}") + + # Сразу после логина — токен работает + code, me = call("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert code == 200, f"token should work immediately, got {code}" + print("immediately after login: 200 OK") + + # Ждём истечения + time.sleep(3) + code, err = call("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert code == 401, f"expired token must give 401, got {code}" + print(f"after TTL: {code} (expected 401)") + print("PASS: expiry — token invalidated after SESSION_TTL_SECONDS") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_idempotent_replay.py b/student_tests/test_idempotent_replay.py new file mode 100644 index 0000000..c6f63e3 --- /dev/null +++ b/student_tests/test_idempotent_replay.py @@ -0,0 +1,154 @@ +"""Повторная доставка ready-задачи: no-op, не создаёт дубли. + +Создаёт attachment, ждёт ready, публикует тот же job повторно, +проверяет, что variants не изменились и объекты в S3 не задублированы. +""" +from __future__ import annotations + +import base64 +import io +import json +import sys +import time +import urllib.error +import urllib.request +import uuid + +BASE = ( + sys.argv[sys.argv.index("--base-url") + 1] + if "--base-url" in sys.argv + else "http://localhost:8080" +) + + +def call(method, path, headers=None, body=None, raw=None, timeout=30): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + data = raw + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=timeout) + payload = resp.read() + return resp.status, json.loads(payload) if payload else None + except urllib.error.HTTPError as e: + payload = e.read() + try: + return e.code, json.loads(payload) if payload else None + except Exception: + return e.code, None + + +def main(): + # 1. Регистрация, логин, чат + username = "replay_" + uuid.uuid4().hex[:10] + password = "replay-pass-1234567890" + code, user = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "Replay", "password": password, + }) + assert code == 201, f"register failed: {code} {user}" + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code}" + token = session["token"] + code, chat = call("POST", "/api/v1/chats", + headers={"Authorization": "Bearer " + token}, + body={"title": "Replay", "member_ids": [user["id"]]}) + assert code == 201, f"chat failed: {code}" + chat_id = chat["id"] + + # 2. Upload + fixture = (Path(__file__).resolve().parent.parent / "tests/fixtures/sample.png").read_bytes() + code, att = call( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}, + raw=fixture, + ) + assert code == 202, f"upload failed: {code}" + att_id = att["id"] + print(f"upload: 202 queued ({att_id})") + + # 3. Ждём ready + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + code, att = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + token}) + if code == 200 and att["status"] == "ready": + break + if code == 200 and att["status"] == "failed": + raise AssertionError(f"failed: {att.get('error_code')}") + time.sleep(0.5) + else: + raise AssertionError("not ready in 60s") + + variants_before = att["variants"] + print(f"ready: {len(variants_before)} variants") + + # 4. Повторная доставка: publish тот же job в broker + import subprocess + # job_id из БД + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-t", "-A", + "-c", f"SELECT job_id FROM jobs WHERE attachment_id = '{att_id}' LIMIT 1"], + capture_output=True, text=True, cwd="/home/user/projects/lab5/mtusi2026lab5", + ) + job_id = result.stdout.strip() + assert job_id, f"job_id not found: {result.stderr}" + print(f"job_id: {job_id}") + + # Опубликовать тот же job через worker (pika) + pub_result = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "worker", + "python3", "-c", f""" +import os, json, pika +params = pika.URLParameters(os.environ['BROKER_URL']) +conn = pika.BlockingConnection(params) +ch = conn.channel() +ch.basic_publish('', 'attachment.process.v1', json.dumps({{ + 'schema_version': 1, 'job_id': '{job_id}', 'type': 'attachment.process.v1', + 'attachment_id': '{att_id}', 'pipeline_version': 1, 'attempt': 1, + 'created_at': '2026-01-01T00:00:00Z' +}})) +conn.close() +print('published') +"""], + capture_output=True, text=True, cwd="/home/user/projects/lab5/mtusi2026lab5", + ) + print(f"replay publish: {pub_result.stdout.strip() or pub_result.stderr.strip()}") + + # 5. Ждём 10 сек, проверяем variants + time.sleep(10) + code, att_after = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + token}) + assert code == 200, f"get after failed: {code}" + variants_after = att_after["variants"] + assert variants_after == variants_before, \ + f"variants changed: {len(variants_before)} -> {len(variants_after)}" + print(f"after replay: {len(variants_after)} variants (unchanged)") + + # 6. Проверяем S3 — количество объектов не изменилось (root, mc ls) + for prefix, ext in (("originals", "png"), ("thumbnails", "jpg"), + ("crops", "jpg"), ("optimized", "webp")): + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", "run", "--rm", + "--entrypoint", "sh", "s3-init", "-c", + f"mc alias set local http://s3:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD >/dev/null && " + f"mc stat local/messenger-attachments/{prefix}/{att_id}.{ext} 2>&1 | head -1"], + capture_output=True, text=True, cwd="/home/user/projects/lab5/mtusi2026lab5", + ) + out = (result.stdout + result.stderr).strip() + assert "Name" in out or "Size" in out, f"missing {prefix}: {out}" + print(f" {prefix}/{att_id}.{ext} exists") + + print("PASS: idempotent replay — variants unchanged, no duplicates") + + +if __name__ == "__main__": + from pathlib import Path + main() \ No newline at end of file diff --git a/student_tests/test_parallel_idempotency.py b/student_tests/test_parallel_idempotency.py new file mode 100644 index 0000000..0a5b714 --- /dev/null +++ b/student_tests/test_parallel_idempotency.py @@ -0,0 +1,115 @@ +"""Параллельная идемпотентность: N потоков посылают один POST с одним ключом. + +Ожидаем: ровно одно сообщение создано, все ответы — одинаковые, +дубликатов в истории нет. +""" +from __future__ import annotations + +import base64 +import json +import sys +import threading +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + +BASE = sys.argv[sys.argv.index("--base-url") + 1] if "--base-url" in sys.argv else "http://localhost:8080" +THREADS = 10 + + +def call(method, path, headers=None, body=None): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=15) + raw = resp.read() + return resp.status, json.loads(raw) if raw else None, dict(resp.headers) + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) if raw else None, dict(e.headers) + except Exception: + return e.code, None, dict(e.headers) + + +def main(): + username = "pario_" + uuid.uuid4().hex[:10] + password = "parallel-pass-1234567890" + + # Регистрация + code, user, _ = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "Parallel", "password": password, + }) + assert code == 201, f"register failed: {code} {user}" + user_id = user["id"] + + # Логин + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session, _ = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code}" + token = session["token"] + + # Чат + code, chat, _ = call("POST", "/api/v1/chats", + headers={"Authorization": "Bearer " + token}, + body={"title": "Parallel test", "member_ids": [user_id]}) + assert code == 201, f"chat failed: {code}" + chat_id = chat["id"] + + # Параллельный POST с одним ключом + idem_key = "parallel-" + uuid.uuid4().hex + barrier = threading.Barrier(THREADS) + results = [] + results_lock = threading.Lock() + + def worker(i): + barrier.wait() # стартуют одновременно + code, body, headers = call( + "POST", f"/api/v1/chats/{chat_id}/messages", + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": idem_key, + }, + body={"text": "parallel-message-same"}, # ← ОДИНАКОВЫЙ текст! + ) + with results_lock: + results.append((code, body)) + + with ThreadPoolExecutor(max_workers=THREADS) as pool: + list(pool.map(worker, range(THREADS))) + + # Анализ + print(f"Results: {len(results)} responses") + codes = {} + for code, _ in results: + codes[code] = codes.get(code, 0) + 1 + print(f"Status codes: {codes}") + + # Проверка: ровно одно сообщение создано + code, page, _ = call("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", + headers={"Authorization": "Bearer " + token}) + assert code == 200, f"history failed: {code}" + messages = page["items"] + print(f"Messages in history: {len(messages)}") + + assert len(messages) == 1, f"expected exactly 1 message, got {len(messages)}" + print("PASS: parallel idempotency — exactly one message created") + + # Проверка: все ответы должны быть одинаковыми + codes = [c for c, _ in results] + bodies = [json.dumps(b, sort_keys=True) for _, b in results if b is not None] + unique_bodies = set(bodies) + print(f"Status codes: {dict((c, codes.count(c)) for c in set(codes))}") + print(f"Unique response bodies: {len(unique_bodies)}") + assert codes.count(201) == THREADS, f"expected all {THREADS} to be 201, got {codes.count(201)}" + assert len(unique_bodies) == 1, f"expected same response body, got {len(unique_bodies)}" + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_races.py b/student_tests/test_races.py new file mode 100644 index 0000000..6087556 --- /dev/null +++ b/student_tests/test_races.py @@ -0,0 +1,116 @@ +"""Тест гонки регистрации: два параллельных POST /users с одним username. + +Ожидаем: один 201, второй 409, в БД ровно один пользователь. +Гонка гарантируется через threading.Barrier — оба запроса стартуют одновременно. +""" +from __future__ import annotations + +import json +import sys +import threading +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + +BASE = sys.argv[sys.argv.index("--base-url") + 1] if "--base-url" in sys.argv else "http://localhost:8080" + + +def post_user(username: str, display_name: str, password: str) -> int: + """Отправляет POST /users, возвращает HTTP-код.""" + body = json.dumps( + {"username": username, "display_name": display_name, "password": password}, + ensure_ascii=False, + ).encode("utf-8") + req = urllib.request.Request( + BASE + "/api/v1/users", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + resp = urllib.request.urlopen(req, timeout=10) + return resp.code + except urllib.error.HTTPError as e: + return e.code + + +def main() -> None: + username = "race_" + uuid.uuid4().hex[:12] + password = "race-pass-1234567890" + barrier = threading.Barrier(2) + + def worker(display_name: str) -> int: + barrier.wait() # старт одновременно + return post_user(username, display_name, password) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(worker, ["Первый", "Второй"])) + + results.sort() + print(f"HTTP codes: {results}") + + assert results == [201, 409], f"expected [201, 409], got {results}" + print(f"PASS: race — one 201, one 409 for username={username}") + +def test_chat_rollback() -> None: + """Попытка создать чат с несуществующим участником → 400, чат не создан.""" + import base64 + + # Создаём пользователя + username = "rollback_" + uuid.uuid4().hex[:8] + password = "rollback-pass-1234567890" + code = post_user(username, "Rollback", password) + assert code == 201, f"failed to create user: {code}" + + # Логинимся + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + req = urllib.request.Request( + BASE + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}, + method="POST", + ) + resp = urllib.request.urlopen(req) + session = json.loads(resp.read()) + token = session["token"] + user_id = session["user"]["id"] + + # Пытаемся создать чат с несуществующим UUID + fake_user = "00000000-0000-4000-8000-000000000000" + body = json.dumps( + {"title": "Bad chat", "member_ids": [user_id, fake_user]}, + ensure_ascii=False, + ).encode("utf-8") + req = urllib.request.Request( + BASE + "/api/v1/chats", + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer " + token, + }, + method="POST", + ) + try: + urllib.request.urlopen(req, timeout=10) + code = 200 + except urllib.error.HTTPError as e: + code = e.code + + assert code == 400, f"expected 400 for unknown member, got {code}" + + # Проверяем, что чат не создан — list chats не должен его содержать + req = urllib.request.Request( + BASE + "/api/v1/chats?limit=100", + headers={"Authorization": "Bearer " + token}, + ) + resp = urllib.request.urlopen(req) + page = json.loads(resp.read()) + titles = [c["title"] for c in page["items"]] + assert "Bad chat" not in titles, f"chat was created despite failure: {titles}" + + print("PASS: rollback — chat not created when member is unknown") + + +if __name__ == "__main__": + main() + test_chat_rollback() \ No newline at end of file diff --git a/student_tests/test_variants.py b/student_tests/test_variants.py new file mode 100644 index 0000000..0b178bc --- /dev/null +++ b/student_tests/test_variants.py @@ -0,0 +1,171 @@ +"""Проверка crop 128×128 и optimized WebP 1280×1280 (lab5 grade 4). + +Запуск: + python3 student_tests/test_variants.py --base-url http://localhost:8080 +""" +from __future__ import annotations + +import base64 +import json +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +BASE = ( + sys.argv[sys.argv.index("--base-url") + 1] + if "--base-url" in sys.argv + else "http://localhost:8080" +) +ROOT = Path(__file__).resolve().parent.parent + + +def call(method, path, headers=None, body=None, raw=None, timeout=30): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + data = raw + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=timeout) + payload = resp.read() + return resp.status, payload, dict(resp.headers) + except urllib.error.HTTPError as e: + payload = e.read() + return e.code, payload, dict(e.headers) + + +def jpeg_size(data: bytes) -> tuple[int, int]: + assert data[:2] == b"\xff\xd8", "not a JPEG" + i = 2 + sof = {0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf} + while i < len(data): + while i < len(data) and data[i] == 0xff: + i += 1 + marker = data[i] + i += 1 + if marker in {0xd9, 0xda}: + break + if marker in {0x01, 0xd8} or 0xd0 <= marker <= 0xd7: + continue + length = int.from_bytes(data[i:i+2], "big") + if marker in sof: + return (int.from_bytes(data[i+5:i+7], "big"), + int.from_bytes(data[i+3:i+5], "big")) + i += length + raise AssertionError("JPEG SOF not found") + + +def webp_size(data: bytes) -> tuple[int, int]: + assert data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not a WebP" + # VP8L (lossless) или VP8 (lossy) + chunk = data[12:16] + if chunk == b"VP8 ": + i = data.index(b"\x9d\x01\x2a", 20) + w = int.from_bytes(data[i+3:i+5], "little") & 0x3fff + h = int.from_bytes(data[i+5:i+7], "little") & 0x3fff + return w, h + if chunk == b"VP8L": + b = data[21:25] + bits = int.from_bytes(b, "little") + w = (bits & 0x3FFF) + 1 + h = ((bits >> 14) & 0x3FFF) + 1 + return w, h + raise AssertionError("unknown WebP chunk") + + +def main(): + username = "variants_" + uuid.uuid4().hex[:10] + password = "variants-pass-1234567890" + code, _, _ = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "Variants", "password": password, + }) + assert code == 201, f"register failed: {code}" + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, payload, _ = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code}" + session = json.loads(payload) + token = session["token"] + user_id = session["user"]["id"] + + code, payload, _ = call("POST", "/api/v1/chats", + headers={"Authorization": "Bearer " + token}, + body={"title": "Variants", "member_ids": [user_id]}) + assert code == 201, f"chat failed: {code}" + chat_id = json.loads(payload)["id"] + + fixture = (ROOT / "tests/fixtures/sample.png").read_bytes() + code, payload, _ = call( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}, + raw=fixture, + ) + assert code == 202, f"upload failed: {code}" + att_id = json.loads(payload)["id"] + print(f"upload: 202 queued ({att_id})") + + # Ждём ready + deadline = time.monotonic() + 60 + att = None + while time.monotonic() < deadline: + code, payload, _ = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + token}) + if code == 200: + att = json.loads(payload) + if att["status"] == "ready": + break + time.sleep(0.5) + assert att and att["status"] == "ready", f"not ready: {att}" + + variants = {v["name"]: v for v in att["variants"]} + print(f"variants: {list(variants.keys())}") + + # --- thumbnail --- + assert "thumbnail" in variants, "missing thumbnail" + code, thumb_bytes, headers = call( + "GET", f"/api/v1/attachments/{att_id}/content?variant=thumbnail", + headers={"Authorization": "Bearer " + token}, + ) + assert code == 200, f"thumbnail download: {code}" + assert headers.get("content-type", "").startswith("image/jpeg") + w, h = jpeg_size(thumb_bytes) + assert (w, h) == (256, 160), f"thumbnail {w}x{h}, expected 256x160" + print(f"PASS: thumbnail 256x160 ({len(thumb_bytes)} bytes)") + + # --- crop --- + assert "crop" in variants, "missing crop" + code, crop_bytes, headers = call( + "GET", f"/api/v1/attachments/{att_id}/content?variant=crop", + headers={"Authorization": "Bearer " + token}, + ) + assert code == 200, f"crop download: {code}" + assert headers.get("content-type", "").startswith("image/jpeg") + w, h = jpeg_size(crop_bytes) + assert (w, h) == (128, 128), f"crop {w}x{h}, expected 128x128" + print(f"PASS: crop 128x128 ({len(crop_bytes)} bytes)") + + # --- optimized --- + assert "optimized" in variants, "missing optimized" + code, opt_bytes, headers = call( + "GET", f"/api/v1/attachments/{att_id}/content?variant=optimized", + headers={"Authorization": "Bearer " + token}, + ) + assert code == 200, f"optimized download: {code}" + assert headers.get("content-type", "").startswith("image/webp") + w, h = webp_size(opt_bytes) + # fixture 320x200 → вписан в 1280×1280 без увеличения → 320×200 + assert (w, h) == (320, 200), f"optimized {w}x{h}, expected 320x200 (no upscale)" + print(f"PASS: optimized WebP {w}x{h} ({len(opt_bytes)} bytes)") + + print("\nPASS: all variants — thumbnail, crop, optimized") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_worker_lifecycle.py b/student_tests/test_worker_lifecycle.py new file mode 100644 index 0000000..4522c08 --- /dev/null +++ b/student_tests/test_worker_lifecycle.py @@ -0,0 +1,145 @@ +"""Демонстрация: worker остановлен → 202 queued → worker запущен → ready. + +Требует docker compose. Запуск: + python3 student_tests/test_worker_lifecycle.py --base-url http://localhost:8080 +""" +from __future__ import annotations + +import base64 +import json +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +BASE = ( + sys.argv[sys.argv.index("--base-url") + 1] + if "--base-url" in sys.argv + else "http://localhost:8080" +) +ROOT = Path(__file__).resolve().parent.parent + + +def call(method, path, headers=None, body=None, raw=None): + data = None + h = dict(headers or {}) + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + data = raw + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=30) + payload = resp.read() + return resp.status, json.loads(payload) if payload else None, dict(resp.headers) + except urllib.error.HTTPError as e: + payload = e.read() + try: + return e.code, json.loads(payload) if payload else None, dict(e.headers) + except Exception: + return e.code, None, dict(e.headers) + + +def compose(*args, check=True): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + check=check, capture_output=True, text=True, cwd=str(ROOT), + ) + + +def main(): + # 1. Регистрация, логин, чат + username = "lifecycle_" + uuid.uuid4().hex[:10] + password = "lifecycle-pass-1234567890" + code, user, _ = call("POST", "/api/v1/users", body={ + "username": username, "display_name": "Lifecycle", "password": password, + }) + assert code == 201, f"register failed: {code} {user}" + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + code, session, _ = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}) + assert code == 201, f"login failed: {code}" + token = session["token"] + code, chat, _ = call("POST", "/api/v1/chats", + headers={"Authorization": "Bearer " + token}, + body={"title": "Lifecycle", "member_ids": [user["id"]]}) + assert code == 201, f"chat failed: {code}" + chat_id = chat["id"] + + # 2. Остановить worker + compose("stop", "worker") + print("worker stopped") + + try: + # 3. Upload — 202 queued + fixture = (ROOT / "tests/fixtures/sample.png").read_bytes() + code, att, _ = call( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}, + raw=fixture, + ) + assert code == 202, f"upload expected 202, got {code}" + assert att["status"] == "queued", f"expected queued, got {att['status']}" + att_id = att["id"] + print(f"upload: 202 queued ({att_id})") + + # 4. Через 3 секунды — всё ещё queued + time.sleep(3) + code, att, _ = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + token}) + assert code == 200, f"get failed: {code}" + assert att["status"] == "queued", f"expected queued, got {att['status']}" + print(f"after 3s: still queued") + + # 5. Запустить worker + try: + compose("up", "-d", "--no-deps", "worker") + except subprocess.CalledProcessError as exc: + print(f"START FAILED: {exc.stderr}", file=sys.stderr) + raise + + # 6. Ждём ready (до 60 сек) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + code, att, _ = call("GET", f"/api/v1/attachments/{att_id}", + headers={"Authorization": "Bearer " + token}) + if code == 200 and att["status"] == "ready": + break + if code == 200 and att["status"] == "failed": + raise AssertionError(f"attachment failed: {att['error_code']}") + time.sleep(0.5) + else: + raise AssertionError("attachment did not become ready in 60s") + + # 7. Проверяем variants + variants = [v for v in att["variants"] if v["name"] == "thumbnail"] + assert len(variants) == 1, f"expected 1 thumbnail, got {len(variants)}" + v = variants[0] + assert (v["width"], v["height"]) == (256, 160), \ + f"thumbnail dimensions wrong: {v['width']}x{v['height']}" + print(f"after worker start: ready, thumbnail {v['width']}x{v['height']}") + + # 8. Скачиваем thumbnail (бинарный ответ) + req = urllib.request.Request( + BASE + f"/api/v1/attachments/{att_id}/content?variant=thumbnail", + headers={"Authorization": "Bearer " + token}, + ) + try: + resp = urllib.request.urlopen(req, timeout=30) + except urllib.error.HTTPError as e: + raise AssertionError(f"thumbnail download failed: {e.code}") + thumb_bytes = resp.read() + assert thumb_bytes[:2] == b"\xff\xd8", "not a JPEG" + print(f"thumbnail downloaded: {len(thumb_bytes)} bytes, JPEG") + finally: + # 9. Гарантированно вернуть worker + compose("up", "-d", "--no-deps", "worker", check=False) + + +if __name__ == "__main__": + main() \ No newline at end of file