diff --git a/.env.example b/.env.example index 2b32655..dcde4c0 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,73 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 -APP_PORT=8080 +APP_PORT=8086 APP_ENV=lab +POSTGRES_DB=messenger +POSTGRES_USER=messenger +# Задай свой локальный пароль перед запуском. Не добавляй .env в Git. +POSTGRES_PASSWORD= SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# REDIS_URL задаётся в compose.yaml; Redis хранит сессии с TTL и кэш профилей на 60 секунд. +LOG_LEVEL=INFO +# DATABASE_URL и REDIS_URL передаёт Compose. При ручном запуске задай их в окружении. +# Остальные параметры и значения по умолчанию: src/core/settings.py (AppSettings). + +# Стартовый администратор; повторный запуск не меняет существующий пароль. +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin-password-123 +ADMIN_DISPLAY_NAME=Администратор + +# APP_ENV=production запрещает учебный ADMIN_PASSWORD. Пароль существующего +# администратора должен совпадать с настройкой; автоматического сброса нет. +# HTTP_MAX_BODY_BYTES=65536 +# HTTP_BODY_TIMEOUT_SECONDS=5 +# HTTP_REQUEST_TIMEOUT_SECONDS=5 +# HTTP_CONCURRENCY_LIMIT=64 +# PASSWORD_HASH_CONCURRENCY=2 + +# Lab3: число реплик и лимиты каждой API-реплики. Наружу доступен только LB. API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +API_CPUS=0.5 +API_MEMORY=512m +DATABASE_POOL_SIZE=10 +POSTGRES_CPUS=1.0 + +# Lab4. Local synthetic credentials; MinIO is accessible only inside Compose. +MINIO_ROOT_USER=messenger +MINIO_ROOT_PASSWORD=local-messenger-s3-password +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=local-messenger-api-password S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +UPLOAD_CONCURRENCY=1 +# Manual API startup additionally needs S3_ENDPOINT=http://host:9000. + +# Automatic reconciliation runs every 300 seconds; default grace is five minutes. +S3_RECONCILE_ACCESS_KEY_ID=messenger-reconciler +S3_RECONCILE_SECRET_ACCESS_KEY=local-messenger-reconciler-password +RECONCILE_GRACE_SECONDS=300 + +# Lab5: separate durable Redis Streams broker and isolated image worker. +BROKER_URL=redis://broker:6379/0 +S3_WORKER_ACCESS_KEY_ID=messenger-worker +S3_WORKER_SECRET_ACCESS_KEY=local-messenger-worker-password +WORKER_CONCURRENCY=1 +WORKER_MEMORY=1g +WORKER_MEMORY_MB=768 +WORKER_CPUS=1.0 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. +JOB_TIMEOUT_SECONDS=30 +JOB_LEASE_SECONDS=120 +JOB_RETRY_BASE_SECONDS=2 +JOB_RETRY_CAP_SECONDS=60 +STAGING_RETENTION_SECONDS=604800 +# Only enable on an isolated fault-test stand, never normal operation. +PIPELINE_FAULT_INJECTION=false + +# Lab6: telemetry is asynchronous and is not an API readiness dependency. +OTEL_ENABLED=true OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_ADMIN_PASSWORD=local-grafana-password-change-me +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9090 +LOKI_PORT=3100 +TEMPO_PORT=3200 +COLLECTOR_METRICS_PORT=8889 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..cafd4fa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Matplotlib SVG paths contain generated trailing separators. +evidence/**/*.svg -whitespace +# Preserve generated measurements byte-for-byte for evidence checksums. +evidence/**/*.csv -text -whitespace diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..2013364 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,13 @@ -## Лабораторная 6 +## Лабораторная 6 — наблюдаемость -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор: **Каменский Андрей, БПИ2301**. Целевая оценка: **4**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Сохранён бизнес-API lab5. Добавлены OTLP SDK API/worker, отдельный Collector, +Prometheus, Loki, Tempo, Grafana provisioning и dashboard. W3C context проходит +через durable outbox/Redis Streams и retry. Есть SLI/SLO, локальные alert rules +и RUNBOOK.md. SDK/Collector имеют ограниченные очереди и память. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Быстрая проверка: `make test test-observability`. +Контролируемые инциденты: `python3 tests/observability.py --base-url http://localhost:8086 --incidents`. +Отчёт: REPORT.md; результаты: evidence/lab6-*. Публичные tests/контракты не изменены. +Трек на оценку 5 не заявлен. diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289..2d78596 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -1,4 +1,4 @@ -name: Skeleton contracts +name: Lab6 contracts and application on: [push, pull_request] permissions: contents: read @@ -9,8 +9,24 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - run: python -m pip install -r requirements-dev.txt - - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + - run: make check validate lint PYTHON=python + application: + needs: contracts + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: python -m pip install -r requirements.txt -r requirements-dev.txt + - run: make verify-lab6 PYTHON=python + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab6-evidence + path: | + evidence/lab6-*.json + evidence/lab6-*.txt diff --git a/.gitignore b/.gitignore index b5a7189..a085fb7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +.ruff_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cfd86cb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.13.0-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt \ + && python -m pip uninstall --yes pip \ + && addgroup --system app \ + && adduser --system --ingroup app app + +COPY --chown=app:app src ./src +COPY --chown=app:app migrations ./migrations +COPY --chown=app:app infra/s3 ./infra/s3 +COPY --chown=app:app alembic.ini ./alembic.ini + +USER app +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; from src.core.settings import settings; urllib.request.urlopen(f'http://127.0.0.1:{settings.app_port}/health/ready', timeout=settings.healthcheck_timeout_seconds)" + +CMD ["python", "-m", "src"] diff --git a/Dockerfile.example b/Dockerfile.example deleted file mode 100644 index e333a12..0000000 --- a/Dockerfile.example +++ /dev/null @@ -1,8 +0,0 @@ -# Teaching outline, not a runnable Dockerfile. Implement your own Dockerfile. -# 1. Choose a runtime/toolchain version and pin the base image tag/digest. -# 2. Copy dependency manifests first; install from the language lockfile. -# 3. Copy source and build. For compiled languages use a separate runtime stage. -# 4. Run the HTTP process as PID 1; listen on 0.0.0.0:8080. -# 5. Add non-root, signals and healthcheck according to the current rubric. -# 6. Pass runtime config/secrets at startup, never COPY .env or secret files. -# This file deliberately has no FROM/CMD: selecting/implementing them is the task. diff --git a/LAB6_GUIDE.md b/LAB6_GUIDE.md new file mode 100644 index 0000000..128d008 --- /dev/null +++ b/LAB6_GUIDE.md @@ -0,0 +1,83 @@ +# Как работает lab6 и что объяснять на защите + +Автор: **Каменский Андрей, БПИ2301**. + +## Что осталось от lab5 + +Картинка полностью передаётся API, проходит базовые проверки и сохраняется в MinIO +в staging/. В одной транзакции PostgreSQL создаются вложение queued и job/outbox. +API отвечает 202. Dispatcher отправляет job в отдельный Redis Streams broker. +Worker получает её, читает исходник из MinIO, создаёт thumbnail, crop и WebP, +сохраняет результат и фиксирует ready. Оригинал не перекодируется. + +RabbitMQ не нужен, потому что Redis Streams даёт хранимый журнал, consumer groups, +pending deliveries и подтверждения. Redis очереди отделён от Redis сессий. +Данные очереди лежат в volume, включены AOF, appendfsync=always и noeviction. + +| Механизм | Что делает у нас | +| --- | --- | +| XADD | Dispatcher добавляет ID задания и служебные поля в Stream | +| XREADGROUP | Worker получает очередную доставку в consumer group | +| Pending Entries List | Хранит доставки, которые consumer ещё не подтвердил | +| XAUTOCLAIM | Позволяет забрать зависшую неподтверждённую доставку | +| XACK | Подтверждает результат после DB COMMIT | +| XDEL | Удаляет уже подтверждённую запись, чтобы Stream не рос бесконечно | +| Outbox в PostgreSQL | Сохраняет намерение отправки, если broker недоступен | +| Lease и DB row lock | Защищают захват и восстанавливают работу после смерти worker | + +Один consumer берёт по одной записи; WORKER_CONCURRENCY задаёт число consumers. +Повтор доставки нормален: используем стабильный job_id и детерминированные ключи +результатов. Ready-дубль подтверждается без нового resize. Временные ошибки +повторяются с backoff/jitter, максимум пять попыток на цикл; постоянные дают failed. +CLI позволяет явно повторить failed и пишет audit. + +## Что нового в шестой + +Теперь каждый HTTP-запрос получает trace. Внутри него есть spans — отдельные +участки работы: db.SELECT/INSERT/UPDATE, s3.PUT/GET, queue.publish, queue.consume, +image.transform. Span содержит начало, конец, родителя и результат. + +После HTTP-ответа worker работает в другом процессе. Чтобы связь не потерялась, +мы сохраняем traceparent/tracestate в job, dispatcher извлекает их, создаёт +producer span и записывает новый carrier в очередь. Worker создаёт consumer span +с этим родителем. Retry сохраняет context попытки перед повторной публикацией. + +Логи http.completed/job.started/job.completed/db.error — структурный JSON. +trace_id связывает их с трассой, span_id указывает конкретную операцию. +request_id связывает HTTP-ответ и соответствующее событие. В telemetry не передаются +Authorization, пароль, текст сообщения, изображение, SQL или filename. + +| Компонент | Роль | +| --- | --- | +| OpenTelemetry SDK | Создаёт spans, counters/histograms и структурные события в API/worker | +| Collector | Принимает OTLP, ограничивает память, группирует и отправляет данные | +| Prometheus | Хранит временные ряды и вычисляет alert rules | +| Loki | Хранит логи, позволяет найти события по trace_id | +| Tempo | Хранит трассы и причинные связи между процессами | +| Grafana | Показывает данные из трёх источников; не заменяет эти хранилища | + +## Почему метрики не содержат chat_id + +Каждое новое значение label создаёт новый временной ряд. UUID чатов или вложений +дали бы неограниченный рост. Поэтому route — шаблон /chats/{chat_id}/messages, +а labels — method, route, status_class, outcome, job_type или queue. +Технические trace/span IDs находятся в логах/трассах, не в labels метрик. + +## Что значит p95 + +95% измерений попали ниже этого времени. Мы собираем histogram buckets и +агрегируем buckets всех API-реплик, затем применяем histogram_quantile. +Усреднять p95 разных реплик или заменять p95 средней задержкой неправильно. + +## Что показать + +1. make test: весь бизнес-API работает с включённой телеметрией. +2. Dashboard в Grafana: свежие RPS/latency, очередь и worker. +3. tests/observability.py: один upload найден в Tempo вместе с producer/consumer, + DB/S3 и связанными логами API/worker. +4. Prometheus /alerts: правила и локальные firing/resolved. +5. RUNBOOK.md: как отличить остановленный worker от ожидания INSERT в БД. + +Недоступная telemetry не участвует в /health/ready. SDK экспортирует в фоне, +очереди ограничены; Collector при длительном отказе backend теряет telemetry, +но не бизнес-задания. Эти ограничения описаны в отчёте. diff --git a/LAB6_TASK.md b/LAB6_TASK.md new file mode 100644 index 0000000..a607238 --- /dev/null +++ b/LAB6_TASK.md @@ -0,0 +1,105 @@ +# Лабораторная 6. Наблюдаемость: OpenTelemetry и Grafana + +Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку. + +Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. + +[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) + +## Откуда начинаем + +Перенесите минимум lab5. Бизнес-API не меняется. Должны оставаться API-реплики, БД, S3, broker и worker. + +**Архитектура:** API/worker → OTLP → OTel Collector → хранилища metrics/logs/traces → Grafana; queue context связывает API и worker. + +## Что сделать + +1. Выберите хранилища: например Prometheus для метрик, Loki для логов, Tempo для трасс. Разрешена другая совместимая комбинация с Grafana; Grafana сама не хранит все три сигнала. +2. Добавьте SDK-инструментацию API и worker и отдельный Collector в общий Compose/Swarm; настройте receivers/processors/exporters и хранение данных. +3. Реализуйте набор сигналов contracts/observability.md и импортируемые dashboard/datasources. Не используйте user_id/chat_id/message_id как labels метрик. +4. Проверьте связь запроса, структурного лога и обработки задания. Секреты, тела сообщений и изображения не должны попадать в telemetry. +5. Создайте нагрузку, внедрите два ограниченных отказа и найдите их причину только с помощью сохранённых наблюдений; приложите доказательства. + +## Оценка «3»: работающий минимум + +- Весь минимум lab5 работает. Collector, хранилища трёх сигналов и Grafana включены в общий воспроизводимый стенд; конфиги и provisioning лежат в Git, persistent volumes сохраняют нужную историю. +- API и worker отдают OTLP в Collector. В Grafana видны: RPS/errors/duration API, успешные/ошибочные jobs и duration worker; доступны структурные логи и реальные трассы API и worker. +- Есть импортируемый dashboard, а не только screenshot. Сохраняются низкокардинальные атрибуты и идентификаторы trace/span в логах; содержимое сообщений, пароли/токены не экспортируются. +- На контролируемом запросе можно найти trace и связанный лог; на job — worker trace и лог. make test проходит при включённой телеметрии. Объяснено назначение каждого хранилища. + +## Оценка «4»: уровень стажёра/джуна + +Весь уровень 3 **и все** пункты ниже: + +- W3C trace context переносится API → broker → worker; видны producer/consumer spans с корректной связью parent или link. По одному upload найдена цепочка до S3/DB результата, включая retry. +- Dashboard показывает p50/p95/p99, долю ошибок, queue depth/возраст старейшей job, DB pool и saturation worker. Приведены запросы и единицы измерения; percentile вычисляется из histogram, не средних. +- Задан проверяемый SLI/SLO и окно; сохранены alert rules минимум для высокого error rate и растущего queue lag. Alerts проверены искусственным отказом; доставка наружу не нужна, достаточно локального состояния firing/resolved. +- Collector имеет memory limiter/batch и ограниченную буферизацию/поведение при недоступном backend. Остановка telemetry-хранилища не ломает сообщения и не вызывает бесконечный рост RAM. Отчёт содержит разбор двух инцидентов и шаги диагностики. + +## Оценка «5»: исследование и доказанный результат + +Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. + +**Трек 1. Цена наблюдаемости.** Сравните telemetry off/on и ≥2 sampling/aggregation настройки при одинаковой нагрузке: latency, CPU/RAM, объём ingest, потери spans. Объясните, какие ошибки и редкие медленные запросы теряются; предложите и проверьте сбалансированную конфигурацию. + +**Трек 2. SLO и обнаружение инцидентов.** Обоснуйте SLO для API и времени до ready, реализуйте burn-rate alerts с несколькими окнами, воспроизведите краткий всплеск и длительную деградацию. Измерьте время обнаружения/восстановления, ложные срабатывания и покажите runbook, по которому другой студент находит причину. + +## Как начать и проверить + +```sh +# Работает прямо в skeleton, Python 3.9+; приложение не запускается: +make check + +# После реализации запустите свой сервер/стенд по REPORT.md. +# По умолчанию тест использует http://localhost:8080. +make test +``` + +`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. + +```sh +# Compose-вариант после реализации: +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождитесь /health/ready; затем make test. +# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +``` + + +Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. + +## Сценарий защиты + +1. С нуля поднять стенд, импортировать provisioning без ручного набора dashboard, запустить smoke и нагрузку. +2. Показать свежие метрики, одну трассу и соответствующий структурный лог API и worker; проверить, что это данные своего приложения. +3. Для 4: пройти от upload до worker в одном trace/связанных traces, остановить worker или ограничить DB, увидеть firing и затем resolved. +4. Остановить telemetry backend, проверить сохранение работы API и ограниченное поведение exporter. Для 5: повторить эксперимент. + +## Что должно быть в PR + +Collector configuration, конфиги backend, Grafana provisioning/dashboards, alert rules по уровню, схема сигналов и отчёт инцидентов. + +Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md). + +## Вопросы на защите + +- Чем лог, метрика и trace отвечают на разные вопросы об одном инциденте? +- Почему chat_id в labels может исчерпать хранилище? +- Как очередь переносит причинную связь между процессами? +- Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик? + +## Первичные материалы + +- [OTel Collector configuration](https://opentelemetry.io/docs/collector/configuration/) +- [OTel propagation](https://opentelemetry.io/docs/concepts/context-propagation/) +- [W3C Trace Context](https://www.w3.org/TR/trace-context/) +- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/) +- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 48b24e7..85601ae 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,4 +1,4 @@ -# Памятка преподавателю — лабораторная 6 +# Памятка преподавателю — лабораторная 5 ## Что подготовлено diff --git a/Makefile b/Makefile index 5d86d17..5e1e24c 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,56 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:8086 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover - +.PHONY: help up down migrate lint format check validate test load test-attachments test-idempotency test-api-regressions test-attachment-failover test-persistence test-failover test-lab5 test-lab5-faults verify-lab6 jobs-failed help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" - + @echo "up: start; test: public lab6 smoke; test-observability: real signals; verify-lab6: quick acceptance" +up: + docker compose up -d --build --wait +down: + docker compose down +migrate: + docker compose run --rm migration +lint: + $(PYTHON) -m ruff check src + $(PYTHON) -m ruff format --check src +format: + $(PYTHON) -m ruff check --fix src + $(PYTHON) -m ruff format src check: $(PYTHON) scripts/check_contract.py - validate: $(PYTHON) scripts/validate_openapi.py - test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) - load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) - +test-attachments: + $(PYTHON) tests/attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-idempotency: + $(PYTHON) tests/idempotency.py --base-url "$(BASE_URL)" +test-api-regressions: + $(PYTHON) tests/api_regressions.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachment-failover: + $(PYTHON) tests/attachment_failover.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) - + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/restart.sh"; exit 2) + $(PYTHON) scripts/test-persistence.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/stop-one.sh"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +test-lab5: + $(PYTHON) tests/lab5.py --base-url "$(BASE_URL)" +test-lab5-faults: + @test -n "$(COMPOSE_PROJECT_NAME)" || (echo "Set COMPOSE_PROJECT_NAME to your isolated test project"; exit 2) + $(PYTHON) tests/lab5.py --faults --project "$(COMPOSE_PROJECT_NAME)" --base-url "$(BASE_URL)" +verify-lab6: + $(PYTHON) scripts/verify-lab6.py +jobs-failed: + docker compose exec -T dispatcher python -m src.pipeline.admin failed + +.PHONY: test-observability +test-observability: + $(PYTHON) tests/observability.py --base-url "$(BASE_URL)" diff --git a/README.md b/README.md index a607238..7a28c65 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,74 @@ -# Лабораторная 6. Наблюдаемость: OpenTelemetry и Grafana +# Лабораторная 6: OpenTelemetry и Grafana -Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку. +Автор: **Каменский Андрей, БПИ2301**. Целевая оценка: **4**. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Мессенджер из lab5 с двумя API-репликами, PostgreSQL, Redis-сессиями, MinIO, +Redis Streams broker и worker. Добавлены реальные метрики, структурные логи, +сквозные трассы, готовый dashboard и локальные alerts. -[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) +[Задание](LAB6_TASK.md) · [Отчёт](REPORT.md) · [Объяснение](LAB6_GUIDE.md) · +[Диагностика инцидентов](RUNBOOK.md) · [Контракт](contracts/observability.md) -## Откуда начинаем +## Запуск -Перенесите минимум lab5. Бизнес-API не меняется. Должны оставаться API-реплики, БД, S3, broker и worker. - -**Архитектура:** API/worker → OTLP → OTel Collector → хранилища metrics/logs/traces → Grafana; queue context связывает API и worker. - -## Что сделать - -1. Выберите хранилища: например Prometheus для метрик, Loki для логов, Tempo для трасс. Разрешена другая совместимая комбинация с Grafana; Grafana сама не хранит все три сигнала. -2. Добавьте SDK-инструментацию API и worker и отдельный Collector в общий Compose/Swarm; настройте receivers/processors/exporters и хранение данных. -3. Реализуйте набор сигналов contracts/observability.md и импортируемые dashboard/datasources. Не используйте user_id/chat_id/message_id как labels метрик. -4. Проверьте связь запроса, структурного лога и обработки задания. Секреты, тела сообщений и изображения не должны попадать в telemetry. -5. Создайте нагрузку, внедрите два ограниченных отказа и найдите их причину только с помощью сохранённых наблюдений; приложите доказательства. - -## Оценка «3»: работающий минимум - -- Весь минимум lab5 работает. Collector, хранилища трёх сигналов и Grafana включены в общий воспроизводимый стенд; конфиги и provisioning лежат в Git, persistent volumes сохраняют нужную историю. -- API и worker отдают OTLP в Collector. В Grafana видны: RPS/errors/duration API, успешные/ошибочные jobs и duration worker; доступны структурные логи и реальные трассы API и worker. -- Есть импортируемый dashboard, а не только screenshot. Сохраняются низкокардинальные атрибуты и идентификаторы trace/span в логах; содержимое сообщений, пароли/токены не экспортируются. -- На контролируемом запросе можно найти trace и связанный лог; на job — worker trace и лог. make test проходит при включённой телеметрии. Объяснено назначение каждого хранилища. - -## Оценка «4»: уровень стажёра/джуна - -Весь уровень 3 **и все** пункты ниже: - -- W3C trace context переносится API → broker → worker; видны producer/consumer spans с корректной связью parent или link. По одному upload найдена цепочка до S3/DB результата, включая retry. -- Dashboard показывает p50/p95/p99, долю ошибок, queue depth/возраст старейшей job, DB pool и saturation worker. Приведены запросы и единицы измерения; percentile вычисляется из histogram, не средних. -- Задан проверяемый SLI/SLO и окно; сохранены alert rules минимум для высокого error rate и растущего queue lag. Alerts проверены искусственным отказом; доставка наружу не нужна, достаточно локального состояния firing/resolved. -- Collector имеет memory limiter/batch и ограниченную буферизацию/поведение при недоступном backend. Остановка telemetry-хранилища не ломает сообщения и не вызывает бесконечный рост RAM. Отчёт содержит разбор двух инцидентов и шаги диагностики. - -## Оценка «5»: исследование и доказанный результат - -Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. - -**Трек 1. Цена наблюдаемости.** Сравните telemetry off/on и ≥2 sampling/aggregation настройки при одинаковой нагрузке: latency, CPU/RAM, объём ingest, потери spans. Объясните, какие ошибки и редкие медленные запросы теряются; предложите и проверьте сбалансированную конфигурацию. - -**Трек 2. SLO и обнаружение инцидентов.** Обоснуйте SLO для API и времени до ready, реализуйте burn-rate alerts с несколькими окнами, воспроизведите краткий всплеск и длительную деградацию. Измерьте время обнаружения/восстановления, ложные срабатывания и покажите runbook, по которому другой студент находит причину. - -## Как начать и проверить +Нужен Docker с Compose v2. При первом запуске образы скачиваются; MinIO собирается +из закреплённого commit. Все компоненты входят в репозиторий. ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check - -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test +python3 scripts/setup-env.py # если .env ещё нет +docker compose up -d --build --wait +python3 tests/smoke.py --base-url http://localhost:8086 +python3 tests/observability.py --base-url http://localhost:8086 ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +- API: http://localhost:8086/docs +- Grafana: http://localhost:3000/d/messenger-lab6 +- Prometheus / локальные alerts: http://localhost:9090/alerts +- Tempo API: http://localhost:3200 +- Loki API: http://localhost:3100 -```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. -``` +Для просмотра Grafana вход не нужен: anonymous Viewer только на loopback. +Для редактирования: admin и GRAFANA_ADMIN_PASSWORD из локального .env. +Секреты setup-env.py создаёт случайно, не печатает и не перезаписывает существующий +.env. Конфиги datasource/dashboard автоматически загружаются из infra/observability/. +Остановить с сохранением данных: `docker compose down`. +Намеренно удалить все учебные данные текущего проекта: `docker compose down --volumes`. -Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): +## Что добавлено + +API/worker → OTLP HTTP/protobuf → Collector → Prometheus / Loki / Tempo → Grafana. +Метрики отвечают «что и насколько ухудшилось», логи — «какое событие произошло», +трасса — «на каком участке потрачено время или возникла ошибка». + +W3C context сохраняется в PostgreSQL job/outbox, передаётся через Redis Stream и +восстанавливается worker. Upload, producer/consumer, DB и S3 находятся в одной +причинной цепочке. При retry следующий producer продолжает consumer предыдущей +попытки. X-Request-Id сохранён; дополнительно API возвращает X-Trace-Id. + +Dashboard показывает RPS, 5xx, p50/p95/p99 из histogram, jobs/outcomes, время до ready, +backlog/возраст, DB pool, worker saturation, CPU/RAM и связанные логи. Labels метрик +не содержат пользовательские ID, имена файлов, токены или сырые URL. + +## Проверки + +Быстрый тест запущенного стенда: `make test test-observability`. +Публичные tests skeleton lab6 и контракты не изменены. ```sh python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt -make validate PYTHON=.venv/bin/python +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate lint PYTHON=.venv/bin/python +make verify-lab6 PYTHON=.venv/bin/python ``` -Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. +Для коротких контролируемых инцидентов уровня «4» на своём учебном стенде: -## Сценарий защиты +```sh +python3 tests/observability.py --base-url http://localhost:8086 --incidents +``` -1. С нуля поднять стенд, импортировать provisioning без ручного набора dashboard, запустить smoke и нагрузку. -2. Показать свежие метрики, одну трассу и соответствующий структурный лог API и worker; проверить, что это данные своего приложения. -3. Для 4: пройти от upload до worker в одном trace/связанных traces, остановить worker или ограничить DB, увидеть firing и затем resolved. -4. Остановить telemetry backend, проверить сохранение работы API и ограниченное поведение exporter. Для 5: повторить эксперимент. - -## Что должно быть в PR - -Collector configuration, конфиги backend, Grafana provisioning/dashboards, alert rules по уровню, схема сигналов и отчёт инцидентов. - -Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md). - -## Вопросы на защите - -- Чем лог, метрика и trace отвечают на разные вопросы об одном инциденте? -- Почему chat_id в labels может исчерпать хранилище? -- Как очередь переносит причинную связь между процессами? -- Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик? - -## Первичные материалы - -- [OTel Collector configuration](https://opentelemetry.io/docs/collector/configuration/) -- [OTel propagation](https://opentelemetry.io/docs/concepts/context-propagation/) -- [W3C Trace Context](https://www.w3.org/TR/trace-context/) -- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/) -- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) +Этот тест временно останавливает worker, блокирует INSERT в messages, проверяет +alerts firing/resolved и работу API при остановленном Tempo. Все воздействия +снимаются в finally. Результаты и реальные traces/logs находятся в evidence/. +По умолчанию verify-lab6 запускает только smoke и проверку сигналов. diff --git a/REPORT.md b/REPORT.md index b97ae42..bce2eec 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,203 @@ -# Отчёт — лабораторная 6 +# Лабораторная 6. Наблюдаемость: OpenTelemetry и Grafana -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: **Каменский Андрей, БПИ2301**. Целевая оценка: **4**. +Исследовательский трек на 5 не заявлен. Реализация продолжает собственную lab5. +Commit/PR в рамках этой работы не публиковались. -## Запуск из чистого clone +## Запуск и проверка -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Docker с Compose v2, Python 3.13 для полного набора инструментов. Все зависимости +поднимаются из этого репозитория. Предварительно установленный broker/БД/S3 не нужен. -## Архитектура и решения +```sh +python3 scripts/setup-env.py +docker compose config --quiet +docker compose up -d --build --wait +python3 tests/smoke.py --base-url http://localhost:8086 +python3 tests/observability.py --base-url http://localhost:8086 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +setup-env.py создаёт .env со случайными паролями и правами 0600, не выводит секреты +и не перезаписывает существующий файл. Первый запуск скачивает образы и компилирует +закреплённый MinIO. Миграции до 010 и bootstrap bucket/политик выполняются автоматически. +010 добавляет traceparent/tracestate к durable jobs, прежние данные сохраняются. -## Доказательства критериев +Grafana: http://localhost:3000/d/messenger-lab6. Просмотр без входа разрешён только +на локальном loopback, права Viewer. Редактор: admin / GRAFANA_ADMIN_PASSWORD из .env. +API: http://localhost:8086/docs. Prometheus и alerts: http://localhost:9090/alerts. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Обычная остановка `docker compose down` сохраняет данные. +Намеренное удаление всех учебных данных выбранного проекта: +`docker compose down --volumes`. Эта команда для обычного запуска не нужна. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Быстрая приёмка: `make verify-lab6 PYTHON=.venv/bin/python` после установки +requirements.txt и requirements-dev.txt. Опциональная проверка инцидентов: +`python3 tests/observability.py --base-url http://localhost:8086 --incidents`. +Тест снимает воздействия в finally. Долгие эксперименты и исследование цены +наблюдаемости не проводились по выбранному уровню и запросу на короткие проверки. -## Эксперименты и отказы +## Компоненты и версии -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Компонент | Версия / роль | +| --- | --- | +| Python / FastAPI / Pillow | 3.13.0 / 0.141.1 / 13.0.0; приложение и изображения | +| PostgreSQL / Redis | 16.6 / 7.4.2; данные, сессии и отдельный Streams broker | +| HAProxy / MinIO | 3.0.12 / закреплённый commit RELEASE.2025-10-15T17-29-55Z | +| OpenTelemetry Python | API/SDK/OTLP HTTP 1.31.1, semconv 0.52b1 | +| Collector contrib | 0.123.0; OTLP receiver, memory_limiter, batch, exporters | +| Prometheus | 3.2.1; временные ряды и вычисление alert rules | +| Loki | 3.4.2; структурные логи через нативный OTLP endpoint | +| Tempo | 2.7.2; реальные traces API/worker | +| Grafana | 11.6.0; импортируемые datasource/dashboard provisioning | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Схема: API/worker/dispatcher → OTLP HTTP/protobuf → Collector → Prometheus / Loki / +Tempo → Grafana. Collector не является долговременным хранилищем, Grafana — интерфейс. +Все backend имеют persistent volumes; Prometheus, Loki и Tempo настроены на 7 дней. +Конфигурация находится в infra/observability/. JSON dashboard содержит запросы и единицы. -## Ограничения и следующие шаги +Бизнес-API lab5 сохранён: пользователи, Bearer-сессии, два API, сообщения/пагинация, +идемпотентность, raw upload 202 queued, durable outbox, очередь, worker, оригинал и +thumbnail/crop/WebP, retry, failed и replay. Публичные tests/контракты skeleton lab6 +не изменены. Дополнительный X-Trace-Id облегчает поиск своего запроса. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Инструментация и перенос контекста + +Каждый HTTP-запрос получает настоящий SERVER span. После маршрутизации его имя и +http.route используют шаблон, а не URL с UUID. SQLAlchemy events создают CLIENT spans +DB-операций без SQL/parameters. S3 transport создаёт CLIENT spans без object key/URL. +Worker создаёт CONSUMER span и дочерний image.transform; dispatcher — PRODUCER. + +При принятии файла текущий W3C context сохраняется вместе с job/outbox в PostgreSQL. +После перезапуска он не зависит от памяти API. Dispatcher извлекает его, создаёт +queue.publish и inject-ит traceparent/tracestate в envelope Redis Streams. +Worker extract-ит carrier и создаёт queue.consume с producer-родителем. +При retry текущий consumer context сохраняется в job; следующая публикация +продолжает ту же причинную цепь. Нет передачи тела картинки или произвольного URL. + +Resource attributes: service.name=messenger-api / messenger-worker / +messenger-dispatcher, service.version=1.6.0, service.instance.id=случайный UUID +жизни процесса, deployment.environment.name=lab. API, worker и dispatcher имеют +разные service.name; две API-реплики различаются instance ID. + +Отдельный безопасный event logger экспортирует JSON с timestamp, severity, +service_name, event, trace_id, span_id; HTTP содержит request_id. Не экспортируются +пароли, Authorization/session tokens, тела сообщений, изображения, filename, +SQL, parameters, S3 keys или signed URLs. В exceptions передаётся только класс +ошибки, без произвольного сообщения/stack trace. SDK exception auto-recording +для бизнес-spans отключён. Внутренние предупреждения SDK не пересылаются в Loki. + +## Сигналы и единицы + +| Контракт | Реальная Prometheus-метрика | Labels / единица | +| --- | --- | --- | +| HTTP requests | http_server_requests_total | method, route template, status_class; count | +| HTTP duration | http_server_duration_seconds_bucket / sum / count | те же labels; seconds | +| Jobs processed/failed | worker_jobs_total | job_type, outcome=ready/retry/failed; attempts count | +| Job processing duration | worker_job_duration_seconds_bucket | job_type/outcome; seconds | +| Acceptance → ready | attachment_ready_duration_seconds_bucket | job_type; seconds | +| Queue depth | queue_depth | queue; число queued + processing в PostgreSQL, включая outbox | +| Oldest job | queue_oldest_age_seconds | queue; seconds от принятия самой старой незавершённой job | +| DB pool | db_pool_used / db_pool_waiting | checked-out connections / ожидающие checkout | +| Worker saturation | worker_busy / worker_capacity | число занятых consumers / конфигурационная ёмкость | +| Process resources | process_memory_rss_bytes / process_cpu_time_seconds_total | текущий RSS bytes / CPU seconds | + +Collector также добавляет ограниченные resource labels. Пользовательские IDs, +request/trace IDs, raw URL и filename не используются в labels метрик. +job_id тоже не label. Технические trace/span IDs находятся только в логах/трассах. + +Histogram boundaries: 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, +30, 60, 120 секунд и +Inf. p95 API, например: + +```promql +histogram_quantile(0.95, + sum by (le) (rate(http_server_duration_seconds_bucket{route=~"/api/v1/.*"}[1m]))) +``` + +Для p50/p99 меняется quantile. Buckets агрегируются по всем репликам ДО quantile. +Средние и средние перцентили реплик не используются. HTTP duration измеряется +до готовности ответа/его заголовков, а не длительности полного download stream. +Backlog age не является histogram processing duration. + +## SLI, SLO и alerts + +SLI = 1 − число 5xx / число всех бизнес-запросов за скользящие **5 минут**. +SLO: **не менее 99%** запросов без server error в таком окне. Бизнес-запросы — +route под /api/v1/, достигшие API; health исключены, 4xx не считаются server error. +Ошибки самого HAProxy до попадания в API этот SLI не измеряет — ограничение стенда. +При отсутствии запросов SLI не следует трактовать как доказанное выполнение SLO. + +Prometheus rules в infra/observability/alerts.yaml, evaluation interval 2 секунды: + +- ApiHighErrorRate: 5xx/all >5% в окне 30 секунд, минимум 3 запроса, for=5 секунд. +- QueueLagGrowing: oldest accepted job >15 секунд и depth>0, for=5 секунд. + +Это короткие учебные диагностические пороги, а не multi-window burn-rate трека 5. +Отправка во внешние каналы отсутствует, проверяется локальное firing/resolved. +Правила содержат описание и ссылку на RUNBOOK.md. Prometheus ALERTS хранит историю +состояний; после восстановления правило становится inactive (resolved). + +## Два инцидента и диагностика + +**1. Worker остановлен.** API принимает файл и сообщение с queued-вложением. +queue_depth/oldest растут, QueueLagGrowing переходит в firing. По сохранённым +наблюдениям: публикация есть, свежего consumer/completion нет, HTTP не деградировал. +Причина локализуется в фоновой обработке. После запуска worker тот же ID доходит +до ready, backlog/age возвращаются в 0, alert resolved. + +Дополнительная краткая пауза S3 демонстрирует retry: consumer получает временный +отказ, следующая попытка сохраняет trace ID upload. После восстановления S3 +видны несколько consumer spans в одной трассе и готовый результат. + +**2. INSERT блокирован PostgreSQL.** Учебная SHARE-блокировка messages задерживает +запись, но совместима с SELECT readiness. Запросы получают ограниченный 503, +появляются ApiHighErrorRate firing, длинные ERROR db.INSERT spans и db.error logs. +Причина — ожидание DB-операции, а не image resize или просто медленный HTTP handler. +После снятия блокировки запись снова даёт 201; по выходу ошибок из окна alert resolved. + +Команды, подтверждения и trace/log evidence сохраняет tests/observability.py. +Сценарий не требует чтения секретов или ручного поиска по Docker stdout. + +## Ограничение буферов и отказ backend + +В SDK spans/logs передаются background processors: queue=256, batch=32, +schedule=500 ms, HTTP timeout=1 s. Метрики экспортируются раз в 2 s, +export timeout=1.5 s; агрегируются ограниченные наборы атрибутов. Нет sync export +в обработчике запроса. При переполнении SDK теряет telemetry, а не блокирует API. + +Collector: memory_limiter 192 MiB, spike 32 MiB при container limit 256 MiB; +batch до 256 записей, timeout 1 s. Для Loki/Tempo sending_queue=64 batch, +2 consumers, HTTP timeout 2 s, retry до 10 s с интервалом 1–3 s. Очереди in-memory: +при длительном отказе/restart возможны потери, они не называются durable telemetry. + +Метрики Collector фиксируют failures/queue. При остановке Tempo бизнес-сообщения +и новый attachment продолжают работать, а exporter показывает ошибки. Это +проверка ограниченного сценария и конфигурационных caps; длительного исследования +роста RAM под многодневной нагрузкой не проводилось. Бизнес-readiness не зависит +от Collector, Grafana, Prometheus, Loki или Tempo. + +## Доказательства и ограничения + +- evidence/lab6-checks.txt — offline/OpenAPI/lint и публичный smoke. +- evidence/lab6-signals.txt — короткая проверка трёх сигналов и dashboard queries. +- evidence/lab6-trace.json, lab6-correlated-logs.json — реальные связанные данные. +- evidence/lab6-incidents.txt, lab6-observability-summary.json — DB/backend проверки; + lab6-worker-incident.json — ранее проверенные остановка worker и retry. +- evidence/lab6-retry-trace.json — цепочка с повтором. +- evidence/lab6-db-incident-trace.json, lab6-db-incident-logs.json — диагностика DB. + +Стек однокопийный учебный, backend и host — точки отказа; нет HA/backups. +Sampling по умолчанию сохраняет все созданные traces; нагрузочных выводов о цене +такого режима нет. worker.capacity — конфигурация, не heartbeat; последнее значение +может временно сохраняться после остановки процесса. Loss telemetry при отказах +допускается и наблюдается, не обещается exactly-once доставка telemetry. + +## Источники и помощь + +Задание/контракты взяты из предоставленного skeleton lab6. + +- https://opentelemetry.io/docs/collector/configuration/ +- https://opentelemetry.io/docs/collector/internal-telemetry/ +- https://opentelemetry.io/docs/concepts/context-propagation/ +- https://redis.io/docs/latest/develop/data-types/streams/ +- https://grafana.com/docs/loki/latest/send-data/otel/ +- https://prometheus.io/docs/practices/histograms/ diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..9b8a145 --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,46 @@ +# Диагностика lab6 + +Автор: Каменский Андрей, БПИ2301. + +## QueueLagGrowing + +Условие: есть принятая незавершённая работа, возраст старейшей больше 15 секунд +не менее 5 секунд. Источник — PostgreSQL jobs, включая ещё не опубликованный outbox. + +1. Grafana: проверить queue depth и oldest age. Рост обоих при работающем API + означает накопление фоновой работы, а не медленный HTTP upload. +2. Loki: найти job.published, затем job.started/job.completed. Отсутствие свежих + consumer events при наличии публикаций указывает на worker/доставку. +3. Tempo: открыть trace upload. Если есть queue.publish, а consumer отсутствует, + обработка не началась. Если есть consumer со статусом ERROR и s3.GET — проверить S3. +4. Проверить процесс `docker compose ps worker`, восстановить `docker compose start worker`. +5. Убедиться, что прежний attachment_id стал ready, backlog/age упали до 0 и alert resolved. + +worker.capacity отражает конфигурацию, а не самостоятельный heartbeat: последнее +значение не доказывает, что процесс всё ещё жив. + +## ApiHighErrorRate + +Условие: 5xx / все бизнес-запросы > 5% за 30 секунд, минимум 3 запроса, +условие сохраняется 5 секунд. Health и 4xx в numerator не входят. + +1. Grafana: проверить 5xx ratio и p95/p99, DB pool used/waiting, CPU. +2. Loki: найти http.completed со status=503 и его trace_id; найти db.error. +3. Tempo: если длительный db.INSERT заканчивается ERROR, проблема в DB-операции. + Это отличается от длительного image.transform в worker или HTTP без DB-задержки. +4. Проверить блокировки/доступность PostgreSQL; снять учебную блокировку, восстановить БД. +5. Отправить сообщение, проверить 201. Дождаться ухода ошибок из окна 30 секунд + и отсутствия alertstate=firing. Prometheus хранит историю ALERTS для просмотра. + +## Backend telemetry недоступен + +Остановка Tempo не должна ломать сообщения или создание вариантов изображения. +Collector продолжает metrics/logs, в traces exporter растут failures/queue. +Максимум 64 batch в очереди exporter, retry ограничен 10 секундами. По исчерпании +данные отбрасываются; это допустимая потеря наблюдений, не потеря приложения. + +Проверить Collector self metrics на localhost:8889/metrics: +`otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity`, +`otelcol_exporter_send_failed_spans_total` и `otelcol_exporter_enqueue_failed_spans_total`. +Конкретное имя/суффикс зависит от закреплённого Collector; исходные строки сохраняет тест. +Восстановить `docker compose start tempo`, выполнить свежий запрос и найти новый trace. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..d3b7004 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,3 @@ +[alembic] +script_location = %(here)s/migrations +prepend_sys_path = . diff --git a/compose.example.yaml b/compose.example.yaml deleted file mode 100644 index dfa1cfb..0000000 --- a/compose.example.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Topology worksheet, deliberately NOT a runnable stack (no application supplied). -# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment. -# Service names below are logical roles, not imposed container names. -x-lab-number: 6 -x-required-roles: - - api - - postgres - - load_balancer - - s3 - - broker - - worker - - otel_collector - - telemetry_storage - - grafana -# TODO: implement real images/builds, network wiring, volumes, healthchecks, -# credentials and bootstrap. Details: contracts/infrastructure.json and README.md. -services: {} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2bb1c7c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,367 @@ +x-logging: &logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-app: &app + build: . + image: ${APP_IMAGE:-mtusi-messenger:lab6} + env_file: ${ENV_FILE:-.env} + logging: *logging + +services: + minio: + build: ./infra/minio + image: ${MINIO_IMAGE:-mtusi-minio:RELEASE.2025-10-15T17-29-55Z} + logging: *logging + mem_limit: 512m + cpus: 1.0 + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-messenger} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-local-messenger-s3-password} + MINIO_BROWSER: "off" + MINIO_API_STALE_UPLOADS_EXPIRY: "24h" + MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: "1h" + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:9000/minio/health/live"] + interval: 2s + timeout: 3s + retries: 60 + start_period: 10s + + s3-bootstrap: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-s3-bootstrap + command: ["python", "-m", "src.core.storage_bootstrap"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-messenger} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-local-messenger-s3-password} + S3_WORKER_ACCESS_KEY_ID: ${S3_WORKER_ACCESS_KEY_ID:-messenger-worker} + S3_WORKER_SECRET_ACCESS_KEY: ${S3_WORKER_SECRET_ACCESS_KEY:-local-messenger-worker-password} + S3_RECONCILE_ACCESS_KEY_ID: ${S3_RECONCILE_ACCESS_KEY_ID:-messenger-reconciler} + S3_RECONCILE_SECRET_ACCESS_KEY: ${S3_RECONCILE_SECRET_ACCESS_KEY:-local-messenger-reconciler-password} + DATABASE_URL: postgresql://unused:unused@postgres/unused + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-local-messenger-api-password} + depends_on: + minio: + condition: service_healthy + healthcheck: + disable: true + + postgres: + logging: *logging + image: postgres:16.6-bookworm + cpus: ${POSTGRES_CPUS:-1.0} + mem_limit: 512m + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Задай POSTGRES_PASSWORD в .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 3s + retries: 30 + + redis: + logging: *logging + image: redis:7.4.2-bookworm + cpus: 0.25 + mem_limit: 128m + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "64mb", "--maxmemory-policy", "noeviction"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + migration: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-migration + command: ["alembic", "upgrade", "head"] + cpus: 0.5 + mem_limit: 256m + environment: + REDIS_URL: &redis_url redis://redis:6379/0 + DATABASE_URL: &database_url postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Задай POSTGRES_PASSWORD в .env}@postgres:5432/${POSTGRES_DB:-messenger} + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + + api: + <<: *app + cpus: ${API_CPUS:-0.5} + mem_limit: ${API_MEMORY:-512m} + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-local-messenger-api-password} + S3_WORKER_ACCESS_KEY_ID: "" + S3_WORKER_SECRET_ACCESS_KEY: "" + # Do not inherit privileged S3 credentials from env_file into API containers. + MINIO_ROOT_USER: "" + MINIO_ROOT_PASSWORD: "" + S3_RECONCILE_ACCESS_KEY_ID: "" + S3_RECONCILE_SECRET_ACCESS_KEY: "" + OTEL_ENABLED: ${OTEL_ENABLED:-true} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-1} + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: *database_url + REDIS_URL: *redis_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + APP_WORKERS: 1 + DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-10} + DATABASE_POOL_TIMEOUT_SECONDS: 2 + DATABASE_CONNECT_TIMEOUT_SECONDS: 1 + DATABASE_COMMAND_TIMEOUT_SECONDS: 3 + DATABASE_STATEMENT_TIMEOUT_MS: 2500 + SHUTDOWN_TIMEOUT_SECONDS: 10 + HTTP_REQUEST_TIMEOUT_SECONDS: ${HTTP_REQUEST_TIMEOUT_SECONDS:-5} + expose: ["8080"] + depends_on: + s3-bootstrap: + condition: service_completed_successfully + migration: + condition: service_completed_successfully + redis: + condition: service_healthy + healthcheck: + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + + reconciler: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-reconciler + command: ["python", "-m", "src.maintenance.reconcile", "--apply", "--interval", "300"] + mem_limit: 256m + cpus: 0.5 + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + DATABASE_URL: *database_url + REDIS_URL: *redis_url + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_RECONCILE_ACCESS_KEY_ID: ${S3_RECONCILE_ACCESS_KEY_ID:-messenger-reconciler} + S3_RECONCILE_SECRET_ACCESS_KEY: ${S3_RECONCILE_SECRET_ACCESS_KEY:-local-messenger-reconciler-password} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-300} + MINIO_ROOT_USER: "" + MINIO_ROOT_PASSWORD: "" + S3_ACCESS_KEY_ID: "" + S3_SECRET_ACCESS_KEY: "" + depends_on: + migration: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 2s + + broker: + image: redis:7.4.2-bookworm + logging: *logging + mem_limit: 256m + cpus: 0.25 + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "192mb", "--maxmemory-policy", "noeviction"] + volumes: + - broker_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + worker: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-worker + command: ["python", "-m", "src.pipeline.worker"] + init: true + mem_limit: ${WORKER_MEMORY:-1g} + cpus: ${WORKER_CPUS:-1.0} + stop_grace_period: 45s + read_only: true + tmpfs: ["/tmp:rw,noexec,nosuid,size=128m"] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: &pipeline_env + DATABASE_URL: *database_url + REDIS_URL: *redis_url + BROKER_URL: redis://broker:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_WORKER_ACCESS_KEY_ID: ${S3_WORKER_ACCESS_KEY_ID:-messenger-worker} + S3_WORKER_SECRET_ACCESS_KEY: ${S3_WORKER_SECRET_ACCESS_KEY:-local-messenger-worker-password} + MINIO_ROOT_USER: "" + MINIO_ROOT_PASSWORD: "" + S3_RECONCILE_ACCESS_KEY_ID: "" + S3_RECONCILE_SECRET_ACCESS_KEY: "" + S3_ACCESS_KEY_ID: "" + S3_SECRET_ACCESS_KEY: "" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + WORKER_MEMORY_MB: ${WORKER_MEMORY_MB:-768} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-120} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_CAP_SECONDS: ${JOB_RETRY_CAP_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-604800} + OTEL_ENABLED: ${OTEL_ENABLED:-true} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + PIPELINE_FAULT_INJECTION: ${PIPELINE_FAULT_INJECTION:-false} + depends_on: + migration: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 3 + + dispatcher: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-dispatcher + command: ["python", "-m", "src.pipeline.dispatcher"] + init: true + mem_limit: 256m + cpus: 0.25 + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: *pipeline_env + depends_on: + migration: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 3 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + command: ["--config=/etc/otelcol/config.yaml"] + mem_limit: 256m + cpus: 0.5 + volumes: + - ./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro + ports: + - "127.0.0.1:${COLLECTOR_METRICS_PORT:-8889}:8888" + + prometheus: + image: prom/prometheus:v3.2.1 + mem_limit: 512m + cpus: 0.5 + command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.retention.time=7d"] + volumes: + - ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"] + + loki: + image: grafana/loki:3.4.2 + mem_limit: 384m + cpus: 0.5 + command: ["-config.file=/etc/loki/config.yaml"] + volumes: + - ./infra/observability/loki.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + ports: ["127.0.0.1:${LOKI_PORT:-3100}:3100"] + + tempo: + image: grafana/tempo:2.7.2 + user: "0:0" + mem_limit: 384m + cpus: 0.5 + command: ["-config.file=/etc/tempo/config.yaml"] + volumes: + - ./infra/observability/tempo.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + ports: ["127.0.0.1:${TEMPO_PORT:-3200}:3200"] + + grafana: + image: grafana/grafana:11.6.0 + mem_limit: 384m + cpus: 0.5 + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + volumes: + - grafana_data:/var/lib/grafana + - ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"] + + lb: + image: haproxy:3.0.12-alpine + logging: *logging + cpus: 0.5 + mem_limit: 128m + read_only: true + tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"] + security_opt: ["no-new-privileges:true"] + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + stop_signal: SIGUSR1 + stop_grace_period: 15s + +volumes: + redis_data: + postgres_data: + minio_data: + broker_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/course-tests/README.md b/course-tests/README.md new file mode 100644 index 0000000..0dfa3f8 --- /dev/null +++ b/course-tests/README.md @@ -0,0 +1,57 @@ +# Публичные проверки + +Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv. + +```sh +make check +make test BASE_URL=http://localhost:8080 +# без make: +python3 tests/smoke.py --base-url http://localhost:8080 +``` + +`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. + +Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются. + +## Проверка сохранности, lab2+ + +Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные. + +```sh +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +Тест создаёт данные и сессию, запускает указанный файл без shell interpolation, ожидает готовности и проверяет те же данные **с прежним токеном**. В lab4+ также проверяется исходный файл. Hook может работать до 90 секунд, готовность затем ожидается до 90 секунд. TTL сессии для этого теста — минимум 300 секунд. Успех не доказывает, что hook действительно перезапустил БД: приложите команды/состояния контейнеров до и после. + +## Проверка отказа, lab3+ + +Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо. + +```sh +chmod +x scripts/stop-one.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Перед отказом тест наблюдает ≥2 API-реплики; 15 секунд отправляет сообщения параллельно выполнению hook, повторяет неопределённый POST с тем же ключом, проверяет восстановление записи за ≤10 секунд, отсутствие остановленной реплики в последних ответах, сохранность подтверждённых сообщений и отсутствие дублей. Ошибки переходного периода учитываются. Затем восстановите реплику и повторите для другой. Скрипт — проверка одного отказа API, не гарантия HA БД/LB/host. Вывод hook скрыт, чтобы не утекли секреты; отлаживайте его отдельно с безопасным выводом. + +## Нагрузка + +```sh +python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +``` + +Это простой **closed-loop** генератор POST сообщений; он ограничен производительностью клиента и страдает coordinated omission. Он выдаёт successful RPS, ошибки и p50/p95/p99 **только успешных запросов**. Во время нагрузки нет прозрачных retry; timeout мог скрыть совершённую запись, поэтому это throughput HTTP-подтверждений, не точный счётчик COMMIT. Изменение размера истории входит в профиль. Для серьёзного исследования используйте k6/Locust/wrk либо свой обоснованный генератор, а не выводите максимальную пропускную способность из одного запуска этого скрипта. Для нагрузки pipeline изображений нужен отдельный сценарий студента. + +## HTTPS, lab7 + +```sh +make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt +make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt +``` + +Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях. + +## Что ещё остаётся доказать + +Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production. diff --git a/course-tests/client.py b/course-tests/client.py new file mode 100644 index 0000000..f0499a2 --- /dev/null +++ b/course-tests/client.py @@ -0,0 +1,217 @@ +"""HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" +import base64 +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import re +import ssl +import time +import urllib.error +import urllib.request +import uuid + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / 'lab.json').read_text())['number'] +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) + +class Failure(AssertionError): + pass + +def check(condition, message): + if not condition: + raise Failure(message) + +def timestamp(value): + check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z') + try: + return dt.datetime.fromisoformat(value[:-1] + '+00:00') + except ValueError as exc: + raise Failure('invalid RFC3339 timestamp') from exc + +def validate(value, schema, where='response'): + """Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI.""" + if '$ref' in schema: + target = SPEC + for part in schema['$ref'].split('/')[1:]: + target = target[part] + return validate(value, target, where) + kinds = schema.get('type', []) + if isinstance(kinds, str): + kinds = [kinds] + matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)} + if kinds: + check(any(matches.get(k,False) for k in kinds), where + ': incorrect type') + if 'enum' in schema: + check(value in schema['enum'], where + ': invalid enum') + if 'const' in schema: + check(value == schema['const'], where + ': invalid const') + if isinstance(value,dict): + props = schema.get('properties',{}) + check(all(k in value for k in schema.get('required',[])), where + ': missing required fields') + if schema.get('additionalProperties') is False: + check(not set(value).difference(props), where + ': unexpected fields') + for key, sub in props.items(): + if key in value: + validate(value[key], sub, where + '.' + key) + if isinstance(value,list): + check(len(value) >= schema.get('minItems',0), where + ': too few items') + check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items') + if schema.get('uniqueItems'): + encoded = [json.dumps(x,sort_keys=True) for x in value] + check(len(encoded)==len(set(encoded)), where + ': duplicate items') + if 'items' in schema: + for item in value: + validate(item, schema['items'], where + '[]') + if isinstance(value,str): + check(len(value) >= schema.get('minLength',0), where + ': too short') + check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long') + if 'pattern' in schema: + check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch') + if schema.get('format') == 'uuid': + try: + uuid.UUID(value) + except ValueError as exc: + raise Failure(where + ': invalid UUID') from exc + if schema.get('format') == 'date-time': + timestamp(value) + if isinstance(value,(int,float)) and not isinstance(value,bool): + check(value >= schema.get('minimum',-float('inf')), where + ': below minimum') + check(value <= schema.get('maximum',float('inf')), where + ': above maximum') + if 'anyOf' in schema: + for alternative in schema['anyOf']: + try: + validate(value, alternative, where) + break + except Failure: + pass + else: + raise Failure(where + ': no anyOf branch matches') + +def shape(value, name): + validate(value, SPEC['components']['schemas'][name]) + return value + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + +class Client: + def __init__(self, base_url=None, ca_file=None, timeout=10): + self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/') + check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)') + check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS') + self.timeout = timeout + context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None) + self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context)) + + def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None): + h = {'Accept':'application/json'} + if actor: + h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token'] + if body is not None: + raw = json.dumps(body,ensure_ascii=False).encode('utf-8') + h['Content-Type'] = 'application/json' + h.update(headers or {}) + req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method) + start = time.monotonic() + try: + response = self.opener.open(req, timeout=self.timeout) + except urllib.error.HTTPError as error: + response = error + except (urllib.error.URLError,TimeoutError,OSError) as exc: + # Do not print URL query, headers, request body, or credentials. + raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None + with response: + status = response.code + rh = response.headers + data = response.read(16*1024*1024 + 1) + check(len(data) <= 16*1024*1024, 'response exceeds test safety limit') + allowed = [expected] if isinstance(expected,int) else expected + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) + check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id') + if LAB>=3: + check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id') + payload = None + if schema or status>=400: + check(rh.get_content_type()=='application/json', 'expected application/json') + try: + payload = json.loads(data) + except (ValueError,UnicodeError): + raise Failure('invalid response JSON') from None + shape(payload, 'Error' if status>=400 else schema) + if status>=400: + check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header') + if status==401 and LAB>=2: + scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer' + check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge') + if status==204: + check(not data, '204 must not contain body') + return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start} + + def user(self, label): + username = 'u_' + label + '_' + uuid.uuid4().hex[:12] + password = 'T3st-' + uuid.uuid4().hex + body = {'username':username,'display_name':'Студент ' + label} + if LAB>=2: + body['password'] = password + user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json'] + check(user['username']==username,'username changed') + actor = {'id':user['id'],'user':user,'registration':body} + if LAB>=2: + credential = base64.b64encode((username+':'+password).encode()).decode() + session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json'] + check(session['user']==user,'session user mismatch') + check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired') + actor['token'] = session['token'] + return actor + + def bootstrap(self): + self.call('GET','/health/live',schema='Health') + self.call('GET','/health/ready',schema='Health') + alice, bob, eve = (self.user(label) for label in ('alice','bob','eve')) + chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] + check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch') + return alice,bob,eve,chat + + def message(self, actor, chat, text, key=None, attachments=None): + body = {'text':text} + if attachments is not None: + body['attachment_ids'] = attachments + headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {} + return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message') + + def attachment(self, actor, chat): + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json'] + check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status') + check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch') + check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch') + return att,content + + def wait_ready(self, actor, att, wait_seconds=60): + deadline = time.monotonic()+wait_seconds + while time.monotonic()=2 and i+length<=len(data),'invalid JPEG segment length') + if marker in sof: + check(length>=8,'invalid JPEG SOF') + return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big')) + i += length + raise Failure('JPEG dimensions not found') diff --git a/course-tests/load.py b/course-tests/load.py new file mode 100644 index 0000000..ad81d59 --- /dev/null +++ b/course-tests/load.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Small closed-loop HTTP write benchmark; not an open-loop capacity proof.""" +from concurrent.futures import ThreadPoolExecutor +import json +import math +from pathlib import Path +import threading +import time +from client import Client, Failure, LAB, arguments, check, run + + +def main(): + def configure(p): + p.add_argument('--duration',type=float,default=30) + p.add_argument('--concurrency',type=int,default=4) + p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens') + args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure) + check(1<=args.duration<=3600,'duration must be 1..3600 seconds') + check(1<=args.concurrency<=128,'concurrency must be 1..128') + c = Client(args.base_url,args.ca_file) + alice,bob,eve,chat = c.bootstrap() + barrier = threading.Barrier(args.concurrency) + def worker(worker_id): + local = Client(args.base_url,args.ca_file,timeout=5) + barrier.wait() + deadline = time.monotonic()+args.duration + latencies,errors = [],0 + index = 0 + while time.monotonic()=2,'persistence requires lab2+') + check(args.mode!='failover' or LAB>=3,'failover requires lab3+') + script = Path(args.action_script).expanduser().resolve() + check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') + check('.example.' not in script.name,'copy and implement the example hook first') + c = Client(args.base_url,args.ca_file,timeout=2) + alice,bob,eve,chat = c.bootstrap() + sent = c.message(alice,chat,'Сохранить при отказе') + original = sent['json'] + target = sent['headers'].get('X-Instance-Id','') + attachment = content = None + if LAB>=4: + attachment,content = c.attachment(alice,chat) + if LAB>=5: + attachment = c.wait_ready(alice,attachment) + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = target + observed = set() + if args.mode=='failover': + for _ in range(40): + observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id']) + check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') + print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') + # No shell interpolation and no Docker commands in this test. + proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + started = time.monotonic() + successes,failures,after_instances = [],0,[] + try: + if args.mode=='persistence': + try: + code = proc.wait(timeout=90) + except subprocess.TimeoutExpired: + raise Failure('restart hook exceeded 90 seconds') from None + check(code==0,'restart hook failed (inspect your script locally)') + deadline = time.monotonic()+90 + while True: + try: + c.call('GET','/health/ready',schema='Health') + c.call('GET','/api/v1/users/me',actor=alice,schema='User') + break + except Failure: + check(time.monotonic()=5,'too few successful writes after stop') + check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') + print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) + # Existing token, no re-login. Read all confirmations and reject duplicates. + messages = history(c,bob,chat) + ids = [m['id'] for m in messages] + check(len(ids)==len(set(ids)),'duplicated message IDs in history') + check(any(m==original for m in messages),'confirmed pre-failure message lost or altered') + for msg in successes: + check(any(m==msg for m in messages),'confirmed in-flight message lost or altered') + # Bodies are unique in this scenario: an uncertain request may exist once, never twice. + texts = [m['text'] for m in messages] + check(len(texts)==len(set(texts)),'uncertain retry created duplicate message') + if attachment: + c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment') + downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob) + check(downloaded['bytes']==content,'attachment lost or changed across failure') + print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.') + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + +if __name__=='__main__': + run(main) diff --git a/course-tests/smoke.py b/course-tests/smoke.py new file mode 100644 index 0000000..4cbfa55 --- /dev/null +++ b/course-tests/smoke.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""One complete public acceptance scenario, cumulative by lab number.""" +import base64 +import hashlib +import uuid +from client import Client, LAB, arguments, check, run, timestamp + + +def main(): + args = arguments('Messenger public smoke; creates synthetic users and messages') + c = Client(args.base_url,args.ca_file) + alice,bob,eve,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id'] + c.call('POST','/api/v1/users',409,body=alice['registration']) + c.call('GET','/api/v1/users/me',401) + c.call('GET',path,404,actor=eve) + me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json'] + check(me==alice['user'],'current user mismatch') + listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json'] + check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat') + hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json'] + check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider') + key = uuid.uuid4().hex + first = c.message(alice,chat,'Привет, мир 👋',key=key)['json'] + second = c.message(bob,chat,'Сообщение Боба')['json'] + check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor') + check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message') + check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed') + page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json'] + check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor') + from urllib.parse import quote + page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json'] + check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history') + ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int)) + check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message') + h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {} + c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h) + c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h) + c.call('GET',path+'/messages?limit=0',400,actor=alice) + if LAB>=2: + bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode() + c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad}) + good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode() + c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good}) + c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']}) + me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json'] + check(me['id']==alice['id'],'X-User-Id overrode bearer identity') + if LAB>=3: + replay = c.message(alice,chat,first['text'],key=key)['json'] + check(replay==first,'idempotency replay differs') + c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key}) + c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'}) + if LAB>=4: + att,content = c.attachment(alice,chat) + # A queued attachment must already be linkable to its own chat. + attached = c.message(alice,chat,'',attachments=[att['id']])['json'] + check(attached['attachment_ids']==[att['id']],'message lost attachment') + ready = c.wait_ready(bob,att) if LAB>=5 else att + check(ready['error_code'] is None,'ready attachment has an error') + download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + check(download['bytes']==content,'download differs from original') + check(download['headers'].get_content_type()=='image/png','original content type mismatch') + for suffix in ('','/content'): + c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve) + if LAB>=5: + variants = [x for x in ready['variants'] if x['name']=='thumbnail'] + check(len(variants)==1,'expected exactly one thumbnail') + v = variants[0] + check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions') + thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob) + check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch') + check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch') + from image_probe import jpeg_size + check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch') + if LAB>=2: + c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) + c.call('GET','/api/v1/users/me',401,actor=alice) + c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice) + print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB) + +if __name__=='__main__': + run(main) diff --git a/evidence/README.md b/evidence/README.md index 8df7001..0cc9649 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,12 @@ -# Доказательства +# Доказательства lab6 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Автор: Каменский Андрей, БПИ2301. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +lab6-checks.txt — короткие проверки структуры, OpenAPI, стиля и публичного API. +lab6-signals.txt — метрики, реальные traces/logs и запросы dashboard. +lab6-incidents.txt и lab6-observability-summary.json — контролируемые отказы. +JSON traces/logs относятся к реальным синтетическим запросам, не к статическим +образцам. Tokens/passwords/body изображения в telemetry не сохраняются. + +lab6-worker-incident.json сохраняет проверку очереди/retry; lab6-incidents.txt — +отдельное продолжение DB/backend после исправления префикса route в метриках. diff --git a/evidence/lab6-checks.txt b/evidence/lab6-checks.txt new file mode 100644 index 0000000..f4ac7b1 --- /dev/null +++ b/evidence/lab6-checks.txt @@ -0,0 +1,10 @@ +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python -m ruff check src +All checks passed! +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python -m ruff format --check src +64 files already formatted +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python tests/smoke.py --base-url "http://localhost:8086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab6-correlated-logs.json b/evidence/lab6-correlated-logs.json new file mode 100644 index 0000000..1c4cbb8 --- /dev/null +++ b/evidence/lab6-correlated-logs.json @@ -0,0 +1,56 @@ +[ + { + "event": "job.completed", + "timestamp": "2026-11-17T15:15:07.104258+00:00", + "severity": "INFO", + "service_name": "messenger-worker", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "6d789cf2190bed03", + "outcome": "ready", + "attempt": 1, + "job_type": "attachment.process.v1", + "duration_seconds": 0.2017499170001429 + }, + { + "event": "job.published", + "timestamp": "2026-11-17T15:15:06.902250+00:00", + "severity": "INFO", + "service_name": "messenger-dispatcher", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "3bfbceb139effbd1", + "attempt": 1, + "job_type": "attachment.process.v1" + }, + { + "event": "job.started", + "timestamp": "2026-11-17T15:15:06.902991+00:00", + "severity": "INFO", + "service_name": "messenger-worker", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "6d789cf2190bed03", + "attempt": 1, + "job_type": "attachment.process.v1" + }, + { + "event": "http.completed", + "timestamp": "2026-11-17T15:15:06.675742+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "801bdc6abdbaba59", + "request_id": "0ecdbe17-1712-4864-ae79-38df19ef2c7d", + "method": "POST", + "route": "/api/v1/chats/{chat_id}/attachments", + "status": 202, + "duration_seconds": 0.1755593749999207 + }, + { + "event": "attachment.accepted", + "timestamp": "2026-11-17T15:15:06.675308+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "801bdc6abdbaba59", + "job_type": "attachment.process.v1" + } +] \ No newline at end of file diff --git a/evidence/lab6-db-incident-logs.json b/evidence/lab6-db-incident-logs.json new file mode 100644 index 0000000..50a7f04 --- /dev/null +++ b/evidence/lab6-db-incident-logs.json @@ -0,0 +1,227 @@ +[ + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.113656+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f1ab8757e130d9b3b8e87acd85115012", + "span_id": "10ca75d8e0ce9007", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.114704+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "69dde9e84908f5b14ab39da9d6b2c30f", + "span_id": "2b1d606367557c42", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.170591+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "98aa6fecf2eda04174aec162d532413f", + "span_id": "4543089015e17a65", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.172345+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "d574895b96f583d31bd53d226301bac6", + "span_id": "59cbe017ec796d04", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.625536+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "4fb197690cd7ed8c34cbaf3af2d2c8c0", + "span_id": "f3b71c918bb75a88", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.671622+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "22d882d3ea564d808dabc96d24e18a37", + "span_id": "b20632277beb3c6b", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.681385+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "314dbd0e393abd2865e38d046f87cf27", + "span_id": "37f57af5302ad41a", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.681488+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "2881ad7651c2ead16b142a1d53525644", + "span_id": "264b3e80956650c9", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.152802+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "2b68f18c2fd49fdebe5b64e69fb8de33", + "span_id": "a71fac67af31936e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.183320+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "40ae54a6e8ba7b0b6b2dfca8de351760", + "span_id": "36b42761727ba91f", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.191879+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "a3d0183572c94e432db07a2efd9a12d1", + "span_id": "ca24eed9905a9bf9", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.191979+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "54b3085f4640962ddffd63936435a4d8", + "span_id": "340c3562a22a467e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.505367+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "acaaafad8ba06b4e7e105cd3f4355987", + "span_id": "38dff7c61d3f6e4d", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.515931+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "c9efffd152bab3e6d2355f7127b3e8d1", + "span_id": "6dd8695d5581baf2", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.575617+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f7364c5486bdc6b3b14b7ef885f00a95", + "span_id": "8db09be701daf82e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.576730+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "ac3e797c761d77bfbf7cda54ff6b02b9", + "span_id": "cdf3e2de745cfc13", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.027064+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "68e11662b15be879a6794f2450b1347b", + "span_id": "780b4cfe086b291e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.074288+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f03afd499dac3f577a1a3a7508b308e9", + "span_id": "cc1f1e6882bbc154", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.085689+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "7e42f82236d190f5e5c5208fcb1f6f39", + "span_id": "0be523a4032a2c36", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.085975+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "829e4fa63c820aa7e3f2de1761f21caf", + "span_id": "4c5aa3f29019e1f6", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.543116+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f6a52507f2922e42155d659bba763970", + "span_id": "5e892c5ae6a001d6", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.590820+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "8f965bae4f6ffa0b3c3cef5c9c52f38e", + "span_id": "6bdafe4d596a25cb", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.595806+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "2f25a5ca948c2aa0c2ef38bd80230d1f", + "span_id": "17b0d785eca64d68", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.595865+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "96bf062721f9b68548202ca761ec56e0", + "span_id": "21a167132ebdb621", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:08:12.863991+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "4df9dbc6eaaaee821bbcc649852367dc", + "span_id": "44727a31e3111cd0", + "error_type": "IntegrityError" + } +] \ No newline at end of file diff --git a/evidence/lab6-db-incident-trace.json b/evidence/lab6-db-incident-trace.json new file mode 100644 index 0000000..197d73d --- /dev/null +++ b/evidence/lab6-db-incident-trace.json @@ -0,0 +1,255 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "CNWCiXdj/44=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510074339418", + "endTimeUnixNano": "1794928512575607086", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "Error" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "jbCb5wHa+C4=", + "name": "POST /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794928509995947085", + "endTimeUnixNano": "1794928512576277919", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "sbSL3OACRAM=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510069821585", + "endTimeUnixNano": "1794928510070631168", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "26YgITrtuvE=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510071310626", + "endTimeUnixNano": "1794928510071793293", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "20d9NgzWuKs=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510072845626", + "endTimeUnixNano": "1794928510073726543", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-incidents.txt b/evidence/lab6-incidents.txt new file mode 100644 index 0000000..b446c07 --- /dev/null +++ b/evidence/lab6-incidents.txt @@ -0,0 +1,5 @@ +{"case": "upload_to_worker_trace_and_logs", "result": "PASS", "trace_id": "fd47457d00c7436da8d99589929e44a5", "spans": 29} +{"case": "metrics_labels_and_dashboard_queries", "result": "PASS", "metrics": 13, "panels": 18} +{"case": "incident_DB_firing", "result": "PASS", "errors": 12, "db_error_spans": 1, "diagnosis": "db.INSERT timeout, matching db.error event; API readiness remains available"} +{"case": "incident_DB_resolved", "result": "PASS"} +{"case": "telemetry_backend_outage_business_survives", "result": "PASS", "messages": 15, "attachment_ready": true, "api_memory": {"bytes": 93831168, "limit": 536870912}, "collector_rss_bytes": 207998976.0, "exporter_failed_spans": 32.0, "max_exporter_queue": 9.0} diff --git a/evidence/lab6-observability-summary.json b/evidence/lab6-observability-summary.json new file mode 100644 index 0000000..c17a352 --- /dev/null +++ b/evidence/lab6-observability-summary.json @@ -0,0 +1,40 @@ +{ + "results": [ + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_DB_firing", + "result": "PASS", + "errors": 12, + "db_error_spans": 1, + "diagnosis": "db.INSERT timeout, matching db.error event; API readiness remains available" + }, + { + "case": "incident_DB_resolved", + "result": "PASS" + }, + { + "case": "telemetry_backend_outage_business_survives", + "result": "PASS", + "messages": 15, + "attachment_ready": true, + "api_memory": { + "bytes": 93831168, + "limit": 536870912 + }, + "collector_rss_bytes": 207998976.0, + "exporter_failed_spans": 32.0, + "max_exporter_queue": 9.0 + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-retry-trace.json b/evidence/lab6-retry-trace.json new file mode 100644 index 0000000..93b98e6 --- /dev/null +++ b/evidence/lab6-retry-trace.json @@ -0,0 +1,1421 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "891b2083-7cf2-4250-afc1-66fd5bf10832" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "ORoZhlAWEcc=", + "parentSpanId": "EjRWeJCrze8=", + "name": "POST /chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794928348113712135", + "endTimeUnixNano": "1794928348132234927", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "4FISSRMyjyA=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348116561010", + "endTimeUnixNano": "1794928348117130677", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "Kt/dbCAsNLo=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348120973302", + "endTimeUnixNano": "1794928348121575052", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "tr2onCZDvXo=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348121955052", + "endTimeUnixNano": "1794928348122239343", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "br9EhJbwD70=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348122602802", + "endTimeUnixNano": "1794928348122931635", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "07LdghSDqyg=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348123745843", + "endTimeUnixNano": "1794928348123956802", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "+AGOk4l4hAo=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348124235593", + "endTimeUnixNano": "1794928348124534385", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "0ji8ZdZwL3w=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348125035052", + "endTimeUnixNano": "1794928348128364302", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "bDIABkUa/CY=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348129024635", + "endTimeUnixNano": "1794928348129832718", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "LZ1mdyNsUAE=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348130217677", + "endTimeUnixNano": "1794928348130606343", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "r7fZ3UZGz3M=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348130731010", + "endTimeUnixNano": "1794928348131034385", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "500f4be3-830d-47c7-9ee4-09a03a1c55f0" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "yh6jZziVMCI=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "queue.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794928348216678718", + "endTimeUnixNano": "1794928348217857885", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "a5Qvy18kFnI=", + "parentSpanId": "yh6jZziVMCI=", + "flags": 768, + "name": "queue.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794928375544746675", + "endTimeUnixNano": "1794928380586867344", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "V2RqMnO14U8=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "s3.GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375576314467", + "endTimeUnixNano": "1794928380580369761", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "GET" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ReadTimeoutError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "iJQ81PnQZ6I=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928380581861344", + "endTimeUnixNano": "1794928380582697303", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "zbrW95wHjRM=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928380583697178", + "endTimeUnixNano": "1794928380584050511", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "q8nGCITS+Es=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928380584809636", + "endTimeUnixNano": "1794928380585355261", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "tUn2q5YA2dM=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375565352134", + "endTimeUnixNano": "1794928375566138509", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "NcYU9EJzEaA=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375567159842", + "endTimeUnixNano": "1794928375567566925", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "9MEws/MDr44=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375568003717", + "endTimeUnixNano": "1794928375568206800", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "mM8SGlmdFQ4=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375568957884", + "endTimeUnixNano": "1794928375569330675", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "BhVvZmgXS5Y=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375569614592", + "endTimeUnixNano": "1794928375569877675", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "z4dCqy3Zu8E=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375570814425", + "endTimeUnixNano": "1794928375570973092", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "O6Ph4KKzAOc=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375571153384", + "endTimeUnixNano": "1794928375571285884", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "500f4be3-830d-47c7-9ee4-09a03a1c55f0" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "pt5rvA0wId0=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 768, + "name": "queue.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794928382403461220", + "endTimeUnixNano": "1794928382405164595", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "RM9Ap7OJJjY=", + "parentSpanId": "pt5rvA0wId0=", + "flags": 768, + "name": "queue.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794928382405238595", + "endTimeUnixNano": "1794928383334647721", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "EcKpHL4zK/k=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382410023262", + "endTimeUnixNano": "1794928383157912596", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "ANuPwb8Mwb8=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794928383158795387", + "endTimeUnixNano": "1794928383323845929", + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "yc7hYRrrwjo=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383324263762", + "endTimeUnixNano": "1794928383327018971", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "Pp8L6QtJ5k4=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383327363554", + "endTimeUnixNano": "1794928383329042512", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "tvr/gpGRHTY=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383329292512", + "endTimeUnixNano": "1794928383330662554", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "BSZky8Gs3Dk=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383330904054", + "endTimeUnixNano": "1794928383332100637", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "0fhbUkQheN4=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383332434179", + "endTimeUnixNano": "1794928383332651554", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "LJwkydWdwDs=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383333278971", + "endTimeUnixNano": "1794928383333652887", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "W33R8I7jDZg=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383333862429", + "endTimeUnixNano": "1794928383334055512", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "zYsu8frY4w4=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382406530970", + "endTimeUnixNano": "1794928382406896137", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "fOhUE9Tn6yM=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382407290179", + "endTimeUnixNano": "1794928382407458595", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "S0LcV/Hc0dA=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382407651220", + "endTimeUnixNano": "1794928382407765345", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "F7TweHMUmTo=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382408011554", + "endTimeUnixNano": "1794928382408135095", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "ACExqAe7JtA=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382408208679", + "endTimeUnixNano": "1794928382408322554", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "m3ZkyxvIqGg=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382408997179", + "endTimeUnixNano": "1794928382409151887", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "MHg3tIi1aEQ=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382409331887", + "endTimeUnixNano": "1794928382409449970", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-signals.txt b/evidence/lab6-signals.txt new file mode 100644 index 0000000..b225308 --- /dev/null +++ b/evidence/lab6-signals.txt @@ -0,0 +1,2 @@ +{"case": "upload_to_worker_trace_and_logs", "result": "PASS", "trace_id": "6127dd4877af4ee8a4c0411427c57ec5", "spans": 29} +{"case": "metrics_labels_and_dashboard_queries", "result": "PASS", "metrics": 13, "panels": 18} diff --git a/evidence/lab6-summary.json b/evidence/lab6-summary.json new file mode 100644 index 0000000..0b3f67f --- /dev/null +++ b/evidence/lab6-summary.json @@ -0,0 +1,71 @@ +{ + "author": "Каменский Андрей, БПИ2301", + "grade_target": 4, + "public_contract_tests_unchanged": true, + "results": [ + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "c4674a56c0144140973ffadebfb865b4", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_worker_firing", + "result": "PASS", + "queue_depth": 1.0, + "oldest_seconds": 22.580556 + }, + { + "case": "retry_keeps_upload_trace", + "result": "PASS", + "trace_id": "6cb20e12f97745b5834e1d4153e363ad", + "consumer_spans": 2 + }, + { + "case": "incident_worker_resolved", + "result": "PASS" + }, + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_DB_firing", + "result": "PASS", + "errors": 12, + "db_error_spans": 1, + "diagnosis": "db.INSERT timeout, matching db.error event; API readiness remains available" + }, + { + "case": "incident_DB_resolved", + "result": "PASS" + }, + { + "case": "telemetry_backend_outage_business_survives", + "result": "PASS", + "messages": 15, + "attachment_ready": true, + "api_memory": { + "bytes": 93831168, + "limit": 536870912 + }, + "collector_rss_bytes": 207998976.0, + "exporter_failed_spans": 32.0, + "max_exporter_queue": 9.0 + } + ] +} diff --git a/evidence/lab6-trace.json b/evidence/lab6-trace.json new file mode 100644 index 0000000..e87e4ba --- /dev/null +++ b/evidence/lab6-trace.json @@ -0,0 +1,1003 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "gBvcar26ulk=", + "parentSpanId": "EjRWeJCrze8=", + "flags": 768, + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794928506500115041", + "endTimeUnixNano": "1794928506675791500", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "kQEe3FRNyYM=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506594712500", + "endTimeUnixNano": "1794928506667958000", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "pzZzbS6ryYE=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506668485791", + "endTimeUnixNano": "1794928506671840708", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "bFI69nFO4Sw=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506672592125", + "endTimeUnixNano": "1794928506673296416", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "XKy+x7V6jO0=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506673925875", + "endTimeUnixNano": "1794928506674299541", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "v0OJJaDrnw4=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506674742041", + "endTimeUnixNano": "1794928506674945666", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "oa5+bdDcECI=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506585396541", + "endTimeUnixNano": "1794928506585931041", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "qliUreo3mc8=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506591477791", + "endTimeUnixNano": "1794928506591835958", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "1FU3fzCjZGc=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506592016125", + "endTimeUnixNano": "1794928506592194208", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "zXtP2PH9gRE=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506593026833", + "endTimeUnixNano": "1794928506593331875", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "v/AWWHDj3DQ=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506593960791", + "endTimeUnixNano": "1794928506594177666", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "a332c142-9ba3-47c0-ab2c-a62df73b9e51" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "O/vOsTnv+9E=", + "parentSpanId": "gBvcar26ulk=", + "flags": 768, + "name": "queue.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794928506899106708", + "endTimeUnixNano": "1794928506902470042", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "3860555f-ed56-4900-ae4a-5e7b82712fdf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "bXic8hkL7QM=", + "parentSpanId": "O/vOsTnv+9E=", + "flags": 768, + "name": "queue.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794928506902902167", + "endTimeUnixNano": "1794928507104380250", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "90NGO4xtu0c=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794928506949619833", + "endTimeUnixNano": "1794928507089492833", + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "0s1HlZQhrAA=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507089944500", + "endTimeUnixNano": "1794928507093279292", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "KvsmUhqSTMg=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507093675667", + "endTimeUnixNano": "1794928507095736417", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "Ej/sw/9+/wM=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507096028958", + "endTimeUnixNano": "1794928507098908333", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "vswUS8KNY9M=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507099219750", + "endTimeUnixNano": "1794928507101115292", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "b4Unm18GieA=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507101459833", + "endTimeUnixNano": "1794928507101694792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "TtuyKW00R88=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507102243167", + "endTimeUnixNano": "1794928507103020792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "JOUymSFGSfM=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507103145167", + "endTimeUnixNano": "1794928507103502958", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "3860555f-ed56-4900-ae4a-5e7b82712fdf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "najEho7C/68=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506930007417", + "endTimeUnixNano": "1794928506930772792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "ZUHBi7/YiIg=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506931914333", + "endTimeUnixNano": "1794928506932305167", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "NSA+BKEPKwA=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506932751875", + "endTimeUnixNano": "1794928506932943708", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "kPC7XrqHWRo=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506933712375", + "endTimeUnixNano": "1794928506934148708", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "RvxyZJmfiKo=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506934736625", + "endTimeUnixNano": "1794928506935103917", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "DAb3TnaXk5g=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506936975917", + "endTimeUnixNano": "1794928506937283792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "CIEq/klKqCU=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506937616625", + "endTimeUnixNano": "1794928506937826250", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "IHEueDpnxnk=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506947425375", + "endTimeUnixNano": "1794928506949196375", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-worker-incident.json b/evidence/lab6-worker-incident.json new file mode 100644 index 0000000..9ecc72f --- /dev/null +++ b/evidence/lab6-worker-incident.json @@ -0,0 +1,32 @@ +{ + "results": [ + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "c4674a56c0144140973ffadebfb865b4", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_worker_firing", + "result": "PASS", + "queue_depth": 1.0, + "oldest_seconds": 22.580556 + }, + { + "case": "retry_keeps_upload_trace", + "result": "PASS", + "trace_id": "6cb20e12f97745b5834e1d4153e363ad", + "consumer_spans": 2 + }, + { + "case": "incident_worker_resolved", + "result": "PASS" + } + ] +} \ No newline at end of file diff --git a/infra/README.md b/infra/README.md index 5a39e86..000faed 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,8 @@ -# Инфраструктура студента +# Инфраструктура lab6 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Автор: Каменский Андрей, БПИ2301. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. - -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +compose.yaml включает приложение, PostgreSQL, Redis-сессии, отдельный Streams broker, +MinIO, worker/dispatcher, HAProxy и наблюдаемость. infra/observability содержит +Collector, Prometheus/alerts, Loki, Tempo и импортируемые Grafana datasources/dashboard. +Все настройки воспроизводимы без ручного создания панелей. Данные сохраняются в volumes. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..5a96573 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,43 @@ +global + log stdout format raw local0 + maxconn 2048 + stats socket /tmp/haproxy.sock mode 600 level admin + stats socket ipv4@127.0.0.1:9999 level admin + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 15s + timeout server 12s + timeout http-request 7s + timeout http-keep-alive 2s + timeout check 1s + # Не повторяем POST автоматически: неизвестный результат повторяет клиент + # с исходным Idempotency-Key. Соединения к недоступному API не ждут долго. + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s + hold refused 1s + accepted_payload_size 8192 + +frontend public + bind :8080 + default_backend api_pool + +backend api_pool + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 + server-template api 1-16 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 0000000..398ae8c --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,20 @@ +# MinIO community distribution is source-only; pin the reviewed release commit. +FROM golang:1.24.8-alpine3.22 AS build +RUN apk add --no-cache git +WORKDIR /source +RUN git init . \ + && git remote add origin https://github.com/minio/minio.git \ + && git fetch --depth 1 origin 9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a \ + && git checkout --detach FETCH_HEAD +ENV CGO_ENABLED=0 GOMAXPROCS=2 +RUN go build -p 2 -trimpath -o /minio . + +FROM alpine:3.22.2 +RUN apk add --no-cache ca-certificates \ + && addgroup -S minio && adduser -S -G minio minio \ + && mkdir /data && chown minio:minio /data +COPY --from=build /minio /usr/local/bin/minio +USER minio +EXPOSE 9000 +ENTRYPOINT ["minio"] +CMD ["server", "/data", "--console-address", ":9001"] diff --git a/infra/observability/alerts.yaml b/infra/observability/alerts.yaml new file mode 100644 index 0000000..eb3c671 --- /dev/null +++ b/infra/observability/alerts.yaml @@ -0,0 +1,23 @@ +groups: + - name: lab6 + interval: 2s + rules: + - alert: ApiHighErrorRate + expr: | + (sum(increase(http_server_requests_total{route=~"/api/v1/.*",status_class="5xx"}[30s])) + / clamp_min(sum(increase(http_server_requests_total{route=~"/api/v1/.*"}[30s])), 1) > 0.05) + and sum(increase(http_server_requests_total{route=~"/api/v1/.*"}[30s])) >= 3 + for: 5s + labels: + severity: warning + annotations: + summary: "Business API 5xx ratio exceeds 5% over 30 seconds" + description: "Exclude health and client errors. Inspect DB spans, db.error logs and pool waiting. See RUNBOOK.md." + - alert: QueueLagGrowing + expr: max(queue_oldest_age_seconds) > 15 and max(queue_depth) > 0 + for: 5s + labels: + severity: warning + annotations: + summary: "Oldest accepted job has waited more than 15 seconds" + description: "Inspect queue depth, worker capacity/busy and job traces. Restore worker and verify lag returns to zero. See RUNBOOK.md." diff --git a/infra/observability/collector.yaml b/infra/observability/collector.yaml new file mode 100644 index 0000000..621ca1b --- /dev/null +++ b/infra/observability/collector.yaml @@ -0,0 +1,67 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 64 + retry_on_failure: + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 64 + retry_on_failure: + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s +extensions: + health_check: + endpoint: 0.0.0.0:13133 +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] diff --git a/infra/observability/grafana/dashboards/messenger.json b/infra/observability/grafana/dashboards/messenger.json new file mode 100644 index 0000000..e4cd41c --- /dev/null +++ b/infra/observability/grafana/dashboards/messenger.json @@ -0,0 +1,519 @@ +{ + "uid": "messenger-lab6", + "title": "Мессенджер · Lab6 · Каменский Андрей", + "schemaVersion": 39, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "tags": [ + "lab6", + "messenger" + ], + "panels": [ + { + "id": 1, + "title": "Business API requests / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_server_requests_total{route=~\"/api/v1/.*\"}[1m]))", + "legendFormat": "RPS", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "id": 2, + "title": "Business API 5xx ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_server_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{route=~\"/api/v1/.*\"}[1m])), 0.00001)", + "legendFormat": "5xx / all business requests", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 3, + "title": "API latency p50 (histogram, all replicas)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "p50", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 4, + "title": "API latency p95 (histogram, all replicas)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 5, + "title": "API latency p99 (histogram, all replicas)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "p99", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 6, + "title": "Jobs per second by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(outcome)(rate(worker_jobs_total[1m]))", + "legendFormat": "{{outcome}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "id": 7, + "title": "Worker processing p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by(le)(rate(worker_job_duration_seconds_bucket[1m])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 8, + "title": "Acceptance to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by(le)(rate(attachment_ready_duration_seconds_bucket[1m])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 9, + "title": "Accepted unfinished jobs (includes outbox)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(queue_depth)", + "legendFormat": "jobs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 10, + "title": "Oldest unfinished job age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(queue_oldest_age_seconds)", + "legendFormat": "age", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 11, + "title": "DB pool used / waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(db_pool_used)", + "legendFormat": "{{service_name}}", + "refId": "A" + }, + { + "expr": "sum by(service_name)(db_pool_waiting)", + "legendFormat": "{{service_name}} waiting", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 12, + "title": "Worker saturation", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(worker_busy) / clamp_min(sum(worker_capacity),1)", + "legendFormat": "busy / capacity", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 13, + "title": "Process resident memory", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(process_memory_rss_bytes)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "id": 14, + "title": "Process CPU cores", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(rate(process_cpu_time_seconds_total[1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 15, + "title": "API availability SLI (5m), SLO 99%", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "1 - (sum(increase(http_server_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min(sum(increase(http_server_requests_total{route=~\"/api/v1/.*\"}[5m])),1)", + "legendFormat": "SLI", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 16, + "title": "Collector export failures / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(exporter)(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "{{exporter}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "id": 17, + "title": "Local firing alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "{{alertname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 99, + "title": "Structured API / worker logs — trace_id links to Tempo", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 24, + "h": 10 + }, + "targets": [ + { + "expr": "{service_name=~\"messenger-(api|worker)\"}", + "refId": "A" + } + ] + } + ] +} diff --git a/infra/observability/grafana/provisioning/dashboards/provider.yaml b/infra/observability/grafana/provisioning/dashboards/provider.yaml new file mode 100644 index 0000000..2079565 --- /dev/null +++ b/infra/observability/grafana/provisioning/dashboards/provider.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Lab6 + type: file + disableDeletion: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/observability/grafana/provisioning/datasources/sources.yaml b/infra/observability/grafana/provisioning/datasources/sources.yaml new file mode 100644 index 0000000..dc4dcd3 --- /dev/null +++ b/infra/observability/grafana/provisioning/datasources/sources.yaml @@ -0,0 +1,34 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 2s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + filterByTraceID: false + customQuery: true + query: '{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"' diff --git a/infra/observability/loki.yaml b/infra/observability/loki.yaml new file mode 100644 index 0000000..5db0331 --- /dev/null +++ b/infra/observability/loki.yaml @@ -0,0 +1,30 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + instance_addr: 127.0.0.1 + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/observability/prometheus.yaml b/infra/observability/prometheus.yaml new file mode 100644 index 0000000..ea5d22d --- /dev/null +++ b/infra/observability/prometheus.yaml @@ -0,0 +1,12 @@ +global: + scrape_interval: 2s + evaluation_interval: 2s +rule_files: + - /etc/prometheus/alerts.yaml +scrape_configs: + - job_name: messenger + static_configs: + - targets: [otel-collector:8889] + - job_name: collector + static_configs: + - targets: [otel-collector:8888] diff --git a/infra/observability/tempo.yaml b/infra/observability/tempo.yaml new file mode 100644 index 0000000..c34ea23 --- /dev/null +++ b/infra/observability/tempo.yaml @@ -0,0 +1,21 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +ingester: + trace_idle_period: 2s + max_block_duration: 1m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/infra/otel/README.md b/infra/otel/README.md deleted file mode 100644 index c46fe89..0000000 --- a/infra/otel/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Конфигурация наблюдаемости — реализовать - -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. - -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. - -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json new file mode 100644 index 0000000..996faf7 --- /dev/null +++ b/infra/s3/api-policy.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::${bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::${bucket}/originals/*", + "arn:aws:s3:::${bucket}/staging/*", + "arn:aws:s3:::${bucket}/variants/*" + ] + } + ] +} diff --git a/infra/s3/reconciler-policy.json b/infra/s3/reconciler-policy.json new file mode 100644 index 0000000..47a8a31 --- /dev/null +++ b/infra/s3/reconciler-policy.json @@ -0,0 +1,28 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:GetBucketVersioning" + ], + "Resource": [ + "arn:aws:s3:::${bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket}/originals/*", + "arn:aws:s3:::${bucket}/staging/*", + "arn:aws:s3:::${bucket}/variants/*" + ] + } + ] +} diff --git a/infra/s3/worker-policy.json b/infra/s3/worker-policy.json new file mode 100644 index 0000000..36ac92e --- /dev/null +++ b/infra/s3/worker-policy.json @@ -0,0 +1,30 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::${bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket}/originals/*", + "arn:aws:s3:::${bucket}/staging/*", + "arn:aws:s3:::${bucket}/variants/*" + ] + } + ] +} diff --git a/lab.json b/lab.json index b365b06..85d21da 100644 --- a/lab.json +++ b/lab.json @@ -4,5 +4,5 @@ "contract_version": "1.6.0", "default_base_url": "http://localhost:8080", "grading": "3; 4 includes 3; 5 includes 4 plus one completed research track", - "scaffold_has_application": false + "scaffold_has_application": true } diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..16eaf7e --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,34 @@ +import asyncio + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import create_async_engine + +from src.core.db.models import Base +from src.core.settings import settings + +url = settings.sqlalchemy_database_url +target_metadata = Base.metadata + + +def run_migrations(connection): + context.configure(connection=connection, target_metadata=target_metadata, compare_type=True) + with context.begin_transaction(): + context.run_migrations() + + +async def run_online(): + engine = create_async_engine(url, poolclass=pool.NullPool) + try: + async with engine.connect() as connection: + await connection.run_sync(run_migrations) + finally: + await engine.dispose() + + +if context.is_offline_mode(): + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + with context.begin_transaction(): + context.run_migrations() +else: + asyncio.run(run_online()) diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..5449f62 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,17 @@ +"""${message}""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/001_initial.py b/migrations/versions/001_initial.py new file mode 100644 index 0000000..b8cef40 --- /dev/null +++ b/migrations/versions/001_initial.py @@ -0,0 +1,60 @@ +"""Пользователи, чаты, участники, сообщения и сессии.""" +from alembic import op +import sqlalchemy as sa + +revision = '001' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'users', + sa.Column('id', sa.Uuid(), primary_key=True), + sa.Column('username', sa.String(32), nullable=False, unique=True), + sa.Column('display_name', sa.String(100), nullable=False), + sa.Column('password_hash', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + sa.CheckConstraint('length(trim(display_name)) > 0'), + ) + op.create_table( + 'chats', + sa.Column('id', sa.Uuid(), primary_key=True), + sa.Column('title', sa.String(100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint('length(trim(title)) > 0'), + ) + op.create_table( + 'chat_members', + sa.Column('chat_id', sa.Uuid(), sa.ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True), + sa.Column('user_id', sa.Uuid(), sa.ForeignKey('users.id'), primary_key=True), + ) + op.create_index('chat_members_user_idx', 'chat_members', ['user_id', 'chat_id']) + op.create_table( + 'messages', + sa.Column('id', sa.Uuid(), primary_key=True), + sa.Column('chat_id', sa.Uuid(), sa.ForeignKey('chats.id'), nullable=False), + sa.Column('sender_id', sa.Uuid(), sa.ForeignKey('users.id'), nullable=False), + sa.Column('text', sa.String(4000), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id', 'sender_id'], ['chat_members.chat_id', 'chat_members.user_id']), + sa.CheckConstraint('length(trim(text)) > 0'), + ) + op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id']) + op.create_table( + 'sessions', + sa.Column('token_hash', sa.Text(), primary_key=True), + sa.Column('user_id', sa.Uuid(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + ) + op.create_index('sessions_expiry_idx', 'sessions', ['expires_at']) + + +def downgrade(): + op.drop_table('sessions') + op.drop_table('messages') + op.drop_table('chat_members') + op.drop_table('chats') + op.drop_table('users') diff --git a/migrations/versions/002_redis_sessions.py b/migrations/versions/002_redis_sessions.py new file mode 100644 index 0000000..109638b --- /dev/null +++ b/migrations/versions/002_redis_sessions.py @@ -0,0 +1,22 @@ +"""Сессии перенесены в Redis.""" +from alembic import op +import sqlalchemy as sa + +revision = '002' +down_revision = '001' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('sessions') + + +def downgrade(): + op.create_table( + 'sessions', + sa.Column('token_hash', sa.Text(), primary_key=True), + sa.Column('user_id', sa.Uuid(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + ) + op.create_index('sessions_expiry_idx', 'sessions', ['expires_at']) diff --git a/migrations/versions/003_admin.py b/migrations/versions/003_admin.py new file mode 100644 index 0000000..7b57ee8 --- /dev/null +++ b/migrations/versions/003_admin.py @@ -0,0 +1,16 @@ +"""Признак системного администратора.""" +from alembic import op +import sqlalchemy as sa + +revision = '003' +down_revision = '002' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('users', sa.Column('is_admin', sa.Boolean(), nullable=False, server_default=sa.false())) + + +def downgrade(): + op.drop_column('users', 'is_admin') diff --git a/migrations/versions/004_idempotency.py b/migrations/versions/004_idempotency.py new file mode 100644 index 0000000..017366a --- /dev/null +++ b/migrations/versions/004_idempotency.py @@ -0,0 +1,25 @@ +"""Общая идемпотентность сообщений, совместимая с предыдущей схемой.""" + +from alembic import op +import sqlalchemy as sa + +revision = "004" +down_revision = "003" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "message_requests", + sa.Column("sender_id", sa.Uuid(), sa.ForeignKey("users.id"), primary_key=True), + sa.Column("chat_id", sa.Uuid(), sa.ForeignKey("chats.id"), primary_key=True), + sa.Column("key", sa.String(128), primary_key=True), + sa.Column("request_text", sa.String(4000), nullable=False), + sa.Column("message_id", sa.Uuid(), sa.ForeignKey("messages.id"), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade(): + op.drop_table("message_requests") diff --git a/migrations/versions/005_message_unicode.py b/migrations/versions/005_message_unicode.py new file mode 100644 index 0000000..762803d --- /dev/null +++ b/migrations/versions/005_message_unicode.py @@ -0,0 +1,55 @@ +"""Сохранять весь допустимый Unicode сообщений, включая U+0000, без потерь.""" + +import sqlalchemy as sa +from alembic import op + +revision = "005" +down_revision = "004" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint("messages_text_check", "messages", type_="check") + op.alter_column( + "messages", + "text", + existing_type=sa.String(4000), + type_=sa.LargeBinary(), + existing_nullable=False, + postgresql_using="convert_to(text, 'UTF8')", + ) + op.alter_column( + "message_requests", + "request_text", + existing_type=sa.String(4000), + type_=sa.LargeBinary(), + existing_nullable=False, + postgresql_using="convert_to(request_text, 'UTF8')", + ) + op.create_check_constraint( + "messages_text_bytes_check", "messages", "octet_length(text) BETWEEN 1 AND 16000" + ) + + +def downgrade(): + # convert_from откажет при наличии U+0000: не удаляем символы или строки + # молча. Откат возможен только если все данные представимы в PostgreSQL text. + op.drop_constraint("messages_text_bytes_check", "messages", type_="check") + op.alter_column( + "messages", + "text", + existing_type=sa.LargeBinary(), + type_=sa.String(4000), + existing_nullable=False, + postgresql_using="convert_from(text, 'UTF8')", + ) + op.alter_column( + "message_requests", + "request_text", + existing_type=sa.LargeBinary(), + type_=sa.String(4000), + existing_nullable=False, + postgresql_using="convert_from(request_text, 'UTF8')", + ) + op.create_check_constraint("messages_text_check", "messages", "length(trim(text)) > 0") diff --git a/migrations/versions/006_unicode_names.py b/migrations/versions/006_unicode_names.py new file mode 100644 index 0000000..e60df49 --- /dev/null +++ b/migrations/versions/006_unicode_names.py @@ -0,0 +1,40 @@ +"""Сохранение допустимого Unicode имён и названий, включая U+0000.""" + +import sqlalchemy as sa +from alembic import op + +revision = "006" +down_revision = "005" +branch_labels = None +depends_on = None + + +def upgrade(): + for table, column in [("users", "display_name"), ("chats", "title")]: + op.drop_constraint(f"{table}_{column}_check", table, type_="check") + op.alter_column( + table, + column, + existing_type=sa.String(100), + type_=sa.LargeBinary(), + existing_nullable=False, + postgresql_using=f"convert_to({column}, 'UTF8')", + ) + op.create_check_constraint( + f"{table}_{column}_bytes_check", table, f"octet_length({column}) BETWEEN 1 AND 400" + ) + + +def downgrade(): + # PostgreSQL откажет при U+0000 и откатит транзакцию вместо потери символов. + for table, column in [("users", "display_name"), ("chats", "title")]: + op.drop_constraint(f"{table}_{column}_bytes_check", table, type_="check") + op.alter_column( + table, + column, + existing_type=sa.LargeBinary(), + type_=sa.String(100), + existing_nullable=False, + postgresql_using=f"convert_from({column}, 'UTF8')", + ) + op.create_check_constraint(f"{table}_{column}_check", table, f"length(trim({column})) > 0") diff --git a/migrations/versions/007_attachments.py b/migrations/versions/007_attachments.py new file mode 100644 index 0000000..e468209 --- /dev/null +++ b/migrations/versions/007_attachments.py @@ -0,0 +1,65 @@ +"""Private S3 attachment metadata and ordered message links.""" + +from alembic import op +import sqlalchemy as sa + +revision = "007" +down_revision = "006" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "attachments", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("chat_id", sa.Uuid(), sa.ForeignKey("chats.id"), nullable=False), + sa.Column("owner_id", sa.Uuid(), sa.ForeignKey("users.id"), nullable=False), + sa.Column("filename", sa.String(255), nullable=False), + sa.Column("object_key", sa.Text(), nullable=False, unique=True), + sa.Column("content_type", sa.String(32), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("chat_id", "id"), + sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760"), + sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')"), + ) + op.create_index("attachments_chat_idx", "attachments", ["chat_id"]) + op.create_unique_constraint("messages_chat_id_id_key", "messages", ["chat_id", "id"]) + op.create_table( + "message_attachments", + sa.Column("message_id", sa.Uuid(), primary_key=True), + sa.Column("attachment_id", sa.Uuid(), primary_key=True), + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("position", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["chat_id", "message_id"], ["messages.chat_id", "messages.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["chat_id", "attachment_id"], ["attachments.chat_id", "attachments.id"] + ), + sa.UniqueConstraint("message_id", "position"), + sa.CheckConstraint("position BETWEEN 0 AND 9"), + ) + op.add_column("message_requests", sa.Column( + "request_attachment_ids", sa.Text(), nullable=False, server_default="" + )) + op.drop_constraint("messages_text_bytes_check", "messages", type_="check") + op.create_check_constraint( + "messages_text_bytes_check", "messages", "octet_length(text) BETWEEN 0 AND 16000" + ) + + +def downgrade(): + # Refuse downgrade if attachment-only messages cannot satisfy the old schema. + op.drop_constraint("messages_text_bytes_check", "messages", type_="check") + op.create_check_constraint( + "messages_text_bytes_check", "messages", "octet_length(text) BETWEEN 1 AND 16000" + ) + op.drop_column("message_requests", "request_attachment_ids") + op.drop_table("message_attachments") + op.drop_constraint("messages_chat_id_id_key", "messages", type_="unique") + op.drop_index("attachments_chat_idx", "attachments") + op.drop_table("attachments") diff --git a/migrations/versions/008_reconciliation.py b/migrations/versions/008_reconciliation.py new file mode 100644 index 0000000..19afaf2 --- /dev/null +++ b/migrations/versions/008_reconciliation.py @@ -0,0 +1,37 @@ +"""Durable upload intents, missing-content marker and reconciliation observations.""" + +import sqlalchemy as sa +from alembic import op + +revision = "008" +down_revision = "007" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("attachments", sa.Column("storage_missing_since", sa.DateTime(timezone=True))) + op.create_table( + "upload_intents", + sa.Column("object_key", sa.Text(), primary_key=True), + sa.Column("state", sa.String(16), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now()), + sa.CheckConstraint("state IN ('pending', 'committed', 'abandoned')"), + ) + op.create_table( + "storage_findings", + sa.Column("object_key", sa.Text(), primary_key=True), + sa.Column("kind", sa.String(16), primary_key=True), + sa.Column("fingerprint", sa.Text()), + sa.Column("first_seen", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False), + sa.Column("resolved_at", sa.DateTime(timezone=True)), + sa.CheckConstraint("kind IN ('orphan', 'missing')"), + ) + + +def downgrade(): + op.drop_table("storage_findings") + op.drop_table("upload_intents") + op.drop_column("attachments", "storage_missing_since") diff --git a/migrations/versions/009_pipeline.py b/migrations/versions/009_pipeline.py new file mode 100644 index 0000000..e3cc77e --- /dev/null +++ b/migrations/versions/009_pipeline.py @@ -0,0 +1,67 @@ +"""Asynchronous image pipeline, durable jobs/outbox and replay audit.""" +import sqlalchemy as sa +from alembic import op + +revision = "009" +down_revision = "008" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("attachments", sa.Column("status", sa.String(16), nullable=False, server_default="ready")) + op.add_column("attachments", sa.Column("error_code", sa.String(64))) + op.add_column("attachments", sa.Column("staging_key", sa.Text())) + op.add_column("attachments", sa.Column("variants", sa.JSON(), nullable=False, server_default="[]")) + op.create_check_constraint("attachments_status_check", "attachments", "status IN ('queued', 'processing', 'ready', 'failed')") + op.create_table( + "attachment_jobs", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False), + sa.Column("pipeline_version", sa.Integer(), nullable=False), + sa.Column("state", sa.String(16), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + sa.Column("cycle_attempt", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("published_at", sa.DateTime(timezone=True)), + sa.Column("lease_until", sa.DateTime(timezone=True)), + sa.Column("claim_token", sa.Uuid()), + sa.Column("error_code", sa.String(64)), + sa.Column("fault", sa.String(32)), + sa.UniqueConstraint("attachment_id", "pipeline_version"), + sa.CheckConstraint("state IN ('queued', 'processing', 'ready', 'failed')"), + sa.CheckConstraint("attempt >= 1 AND cycle_attempt >= 1"), + sa.CheckConstraint("pipeline_version = 1"), + ) + op.create_index("attachment_jobs_dispatch_idx", "attachment_jobs", ["state", "next_run_at"]) + op.create_table( + "job_replays", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False), + sa.Column("operator", sa.String(100), nullable=False), + sa.Column("reason", sa.String(500), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + ) + + # Existing lab4 originals remain readable by the worker as their staging + # source. Retention must never delete this shared source/final key. + op.execute(""" + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, state, attempt, cycle_attempt, + created_at, updated_at, next_run_at) + SELECT gen_random_uuid(), id, 1, 'queued', 1, 1, + clock_timestamp(), clock_timestamp(), clock_timestamp() + FROM attachments + """) + op.execute("UPDATE attachments SET staging_key=object_key, status='queued', updated_at=clock_timestamp()") + + +def downgrade(): + op.drop_table("job_replays") + op.drop_table("attachment_jobs") + op.drop_constraint("attachments_status_check", "attachments") + for name in ("variants", "staging_key", "error_code", "status"): + op.drop_column("attachments", name) diff --git a/migrations/versions/010_trace_context.py b/migrations/versions/010_trace_context.py new file mode 100644 index 0000000..0a7f5c2 --- /dev/null +++ b/migrations/versions/010_trace_context.py @@ -0,0 +1,15 @@ +"""Persist W3C context across durable outbox, retry and process restarts.""" +import sqlalchemy as sa +from alembic import op +revision = "010" +down_revision = "009" +branch_labels = None +depends_on = None + +def upgrade(): + op.add_column('attachment_jobs', sa.Column('traceparent', sa.String(128))) + op.add_column('attachment_jobs', sa.Column('tracestate', sa.String(512))) + +def downgrade(): + op.drop_column('attachment_jobs','tracestate') + op.drop_column('attachment_jobs','traceparent') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6c70477 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.ruff] +target-version = "py313" +line-length = 100 +include = ["src/**/*.py"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103dd..b36fdc9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ -# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +ruff==0.14.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a93addf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,25 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.141.1 +h11==0.16.0 +idna==3.20 +pydantic==2.13.5 +pydantic-core==2.46.5 +starlette==1.7.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +uvicorn==0.53.0 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==25.1.0 +SQLAlchemy[asyncio]==2.0.44 +alembic==1.17.0 +pydantic-settings==2.15.0 +minio==7.2.20 +Pillow==13.0.0 + +opentelemetry-api==1.31.1 +opentelemetry-sdk==1.31.1 +opentelemetry-exporter-otlp-proto-http==1.31.1 diff --git a/scripts/replicas.py b/scripts/replicas.py new file mode 100644 index 0000000..e8cbc36 --- /dev/null +++ b/scripts/replicas.py @@ -0,0 +1,56 @@ +"""Управление только API-контейнерами текущего Compose-проекта.""" + +import json +import os +import subprocess +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def docker(*args): + return subprocess.check_output(["docker", *args], cwd=ROOT, text=True).strip() + + +def compose(*args): + return docker("compose", "--env-file", os.environ.get("ENV_FILE", ".env"), *args) + + +def containers(): + return compose("ps", "--all", "-q", "api").split() + + +def describe(container): + # Никогда не выводим Config.Env: там credentials. + return json.loads(docker("inspect", "--format", "{{json .State}}", container)) + + +def instance_id(container): + code = "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2); print(r.headers['X-Instance-Id'])" + return docker("exec", container, "python", "-c", code) + + +def wait_ready(container, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + state = describe(container) + if state.get("Health", {}).get("Status") == "healthy": + return instance_id(container) + time.sleep(0.5) + raise RuntimeError("API container did not become healthy: " + container[:12]) + + +def wait_visible(instance, timeout=30): + base = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base + "/health/ready", timeout=2) as response: + if response.headers.get("X-Instance-Id") == instance: + return + except OSError: + pass + time.sleep(0.1) + raise RuntimeError("New replica was not discovered by HAProxy") diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh deleted file mode 100755 index 967e951..0000000 --- a/scripts/restart.example.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eu -# TODO: copy to restart.sh and implement using YOUR compose/stack services. -# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable). -# Preserve persistent volumes and schema/data; never down -v / flush / re-seed. -# Allow operator unseal if required in lab7; do not print credentials. -# Exit 0 only after the intended action; test waits for API readiness separately. -printf '%s -' 'TODO: implement restart.sh for your infrastructure' >&2 -exit 2 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..5da325c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Полный restart работающих ролей; volumes не удаляются. +docker compose --env-file "${ENV_FILE:-.env}" restart postgres redis minio broker +docker compose --env-file "${ENV_FILE:-.env}" up -d --wait --force-recreate api +docker compose --env-file "${ENV_FILE:-.env}" restart reconciler dispatcher worker lb diff --git a/scripts/setup-env.py b/scripts/setup-env.py new file mode 100644 index 0000000..2ff4f50 --- /dev/null +++ b/scripts/setup-env.py @@ -0,0 +1,28 @@ +"""Create local synthetic credentials without overwriting an existing .env.""" +import argparse +import os +import secrets +from pathlib import Path + +root=Path(__file__).resolve().parents[1] +parser=argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output',type=Path,default=root/'.env') +args=parser.parse_args() +values={ + 'GRAFANA_ADMIN_PASSWORD':secrets.token_hex(24), + 'POSTGRES_PASSWORD':secrets.token_hex(24), + 'ADMIN_PASSWORD':secrets.token_hex(24), + 'MINIO_ROOT_PASSWORD':secrets.token_hex(24), + 'S3_SECRET_ACCESS_KEY':secrets.token_hex(24), + 'S3_RECONCILE_SECRET_ACCESS_KEY':secrets.token_hex(24), + 'S3_WORKER_SECRET_ACCESS_KEY':secrets.token_hex(24), +} +lines=(root/'.env.example').read_text().splitlines() +content='\n'.join(key+'='+values[key] if '=' in line and (key:=line.split('=',1)[0]) in values else line for line in lines)+'\n' +try: + descriptor=os.open(args.output,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +except FileExistsError: + parser.exit(1,'Файл уже существует; его настройки и пароли сохранены.\n') +with os.fdopen(descriptor,'w') as stream: + stream.write(content) +print('Создан файл локальных настроек с новыми паролями; значения не выводятся.') diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh deleted file mode 100755 index ae149e8..0000000 --- a/scripts/stop-one.example.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eu -# TODO: copy to stop-one.sh, map TARGET_INSTANCE_ID to ONE API container/task. -# Stop that instance and leave it stopped; no stop of DB/LB/volumes. -# Return quickly (within 10 s); the test sends requests during this script. -# Restore the replica explicitly after the test. Never print credentials. -: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" -printf '%s -' 'TODO: implement stop-one.sh for your infrastructure' >&2 -exit 2 diff --git a/scripts/stop-one.py b/scripts/stop-one.py new file mode 100644 index 0000000..0430d67 --- /dev/null +++ b/scripts/stop-one.py @@ -0,0 +1,34 @@ +"""Сопоставить X-Instance-Id реальному контейнеру и оставить его остановленным.""" + +import json +import os +import time + +from replicas import containers, describe, docker, instance_id + +target = os.environ["TARGET_INSTANCE_ID"] +running = [c for c in containers() if describe(c)["Running"]] +if len(running) < 2: + raise SystemExit("Need at least two running API containers") +for container in running: + if instance_id(container) == target: + before = describe(container) + started = time.monotonic() + docker("stop", "--time", "10", container) + after = describe(container) + assert not after["Running"] + print( + json.dumps( + { + "container": container[:12], + "instance_id": target, + "started_at": before["StartedAt"], + "finished_at": after["FinishedAt"], + "stop_seconds": round(time.monotonic() - started, 3), + "exit_code": after["ExitCode"], + } + ) + ) + break +else: + raise SystemExit("Requested instance not found in this Compose project") diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..383f91e --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec "${PYTHON:-python3}" scripts/stop-one.py diff --git a/scripts/test-persistence.py b/scripts/test-persistence.py new file mode 100644 index 0000000..5e2c0c9 --- /dev/null +++ b/scripts/test-persistence.py @@ -0,0 +1,46 @@ +"""Исходный persistence-сценарий плюс доказательство рестарта API и всех хранилищ.""" +import argparse +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def states(): + result = {} + for service in ('api', 'postgres', 'redis', 'minio', 'broker', 'worker', 'dispatcher', 'reconciler', 'lb'): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).split() + if not ids: + raise RuntimeError('Не запущен сервис ' + service) + result[service] = {} + for container in ids: + state = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', container], cwd=ROOT, text=True)) + result[service][container] = {'StartedAt': state['StartedAt'], 'Pid': state['Pid']} + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--action-script', required=True) + parser.add_argument('--ca-file') + args = parser.parse_args() + before = states() + print('ДО: ' + json.dumps(before), flush=True) + command = [sys.executable, str(ROOT / 'tests/resilience.py'), '--mode', 'persistence', + '--action-script', args.action_script, '--base-url', args.base_url] + if args.ca_file: + command += ['--ca-file', args.ca_file] + subprocess.run(command, cwd=ROOT, check=True) + after = states() + print('ПОСЛЕ: ' + json.dumps(after), flush=True) + for service in before: + if before[service] == after[service]: + raise AssertionError('Сервис не перезапущен: ' + service) + print('PASS: StartedAt изменился у API, PostgreSQL, Redis, MinIO, broker, worker, dispatcher, reconciler и LB; прежний токен и данные сохранены') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify-lab6.py b/scripts/verify-lab6.py new file mode 100644 index 0000000..a64c16f --- /dev/null +++ b/scripts/verify-lab6.py @@ -0,0 +1,20 @@ +"""Short acceptance: structure, public API and real telemetry. Incidents are opt-in.""" +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--incidents',action='store_true') +args=p.parse_args() + +def run(command):subprocess.run(command,cwd=ROOT,check=True) + +if not (ROOT/'.env').exists():run([sys.executable,'scripts/setup-env.py']) +run(['make','check','validate','lint','PYTHON='+sys.executable]) +run(['docker','compose','up','-d','--build','--wait']) +base=os.environ.get('BASE_URL','http://localhost:8086') +run([sys.executable,'tests/smoke.py','--base-url',base]) +run([sys.executable,'tests/observability.py','--base-url',base]+(['--incidents'] if args.incidents else [])) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..da5d2b9 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,13 @@ +import uvicorn + +from src.core.settings import settings + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=settings.app_host, + port=settings.app_port, + workers=settings.app_workers, + log_level=settings.log_level.lower(), + timeout_graceful_shutdown=settings.shutdown_timeout_seconds, + ) diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/dependencies.py b/src/api/dependencies.py new file mode 100644 index 0000000..d04ea93 --- /dev/null +++ b/src/api/dependencies.py @@ -0,0 +1,106 @@ +import base64 +import binascii +import re +from typing import Annotated + +from fastapi import Depends, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from fastapi.security.http import HTTPBase, HTTPBasicCredentials +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.session import get_db_session +from src.core.errors import ApiError +from src.core.settings import settings +from src.service.auth import AuthService +from src.service.chat import ChatService +from src.service.dto import UserDTO +from src.service.exceptions import InvalidSessionException +from src.service.message import MessageService +from src.service.user import UserService + +bearer_scheme = HTTPBearer( + scheme_name="bearerAuth", + description="Вставьте токен из ответа на вход, без префикса Bearer.", + auto_error=False, +) +basic_scheme = HTTPBase(scheme="basic", scheme_name="basicAuth", auto_error=False) + + +async def get_basic_credentials( + request: Request, + credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(basic_scheme)], +) -> HTTPBasicCredentials: + if await request.body(): + raise ApiError(400, "invalid_request", "Вход не принимает тело запроса") + try: + authorization = request.headers.getlist("authorization") + if len(authorization) != 1 or len(authorization[0]) > settings.authorization_max_length: + raise ValueError + if credentials is None or credentials.scheme.lower() != "basic": + raise ValueError + decoded = base64.b64decode(credentials.credentials, validate=True).decode("utf-8") + username, separator, password = decoded.partition(":") + if ( + not separator + or re.fullmatch(r"[a-z0-9_]{3,32}", username) is None + or not 12 <= len(password) <= 128 + or not password.strip() + ): + raise ValueError + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "Неверный логин или пароль") from None + return HTTPBasicCredentials(username=username, password=password) + + +def get_user_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> UserService: + return UserService(db_session=db_session) + + +def get_chat_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> ChatService: + return ChatService(db_session=db_session) + + +def get_message_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> MessageService: + return MessageService(db_session=db_session) + + +def get_auth_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> AuthService: + return AuthService( + db_session=db_session, + session_ttl_seconds=settings.session_ttl_seconds, + ) + + +def get_bearer_token( + request: Request, + credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(bearer_scheme)], +) -> str: + if credentials is None or len(request.headers.getlist("authorization")) != 1: + raise ApiError(401, "unauthorized", "Требуется Bearer-токен") + token = credentials.credentials + if ( + credentials.scheme.lower() != "bearer" + or not token + or len(token) > 512 + or any(c.isspace() for c in token) + ): + raise ApiError(401, "unauthorized", "Требуется Bearer-токен") + return token + + +async def get_current_user( + auth: Annotated[AuthService, Depends(get_auth_service)], + token: Annotated[str, Depends(get_bearer_token)], +) -> UserDTO: + try: + return await auth.get_user(token=token) + except InvalidSessionException as error: + raise ApiError(401, "unauthorized", str(error)) from error diff --git a/src/api/router.py b/src/api/router.py new file mode 100644 index 0000000..7171012 --- /dev/null +++ b/src/api/router.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter + +from src.api.v1.routers.attachments import router as attachments_router +from src.api.v1.routers.auth.router import router as auth_router +from src.api.v1.routers.chats.router import router as chats_router +from src.api.v1.routers.messages.router import router as messages_router +from src.api.v1.routers.users.router import router as users_router + +router = APIRouter(prefix="/api/v1", tags=["Версия 1"]) + +router.include_router(users_router) +router.include_router(chats_router) +router.include_router(messages_router) + +router.include_router(auth_router) + +router.include_router(attachments_router) diff --git a/src/api/v1/__init__.py b/src/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/__init__.py b/src/api/v1/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/attachments.py b/src/api/v1/routers/attachments.py new file mode 100644 index 0000000..a5ead39 --- /dev/null +++ b/src/api/v1/routers/attachments.py @@ -0,0 +1,134 @@ +from datetime import datetime +from typing import Annotated, Literal +from uuid import UUID + +from anyio import to_thread +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.dependencies import get_current_user +from src.core.db.models import Attachment +from src.core.db.session import get_db_session +from src.core.errors import ApiError +from src.core.settings import settings +from src.core.storage import STORAGE_ERRORS, get_storage, unavailable +from src.service.attachment import AttachmentService +from src.service.dto import UserDTO + +router = APIRouter(tags=["Вложения"]) + + +class VariantOut(BaseModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/webp"] + size_bytes: int + sha256: str + width: int + height: int + + +class AttachmentOut(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[VariantOut] = Field(default_factory=list) + error_code: str | None = None + created_at: datetime + updated_at: datetime + + +def to_output(attachment: Attachment) -> AttachmentOut: + return AttachmentOut.model_validate(attachment, from_attributes=True) + + +def get_service( + session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> AttachmentService: + return AttachmentService(session) + + +@router.post( + "/chats/{chat_id}/attachments", + status_code=202, + openapi_extra={ + "requestBody": { + "required": True, + "content": { + mime: {"schema": {"type": "string", "format": "binary"}} + for mime in ("image/png", "image/jpeg") + }, + } + }, +) +async def upload_attachment( + chat_id: UUID, + request: Request, + user: Annotated[UserDTO, Depends(get_current_user)], + service: Annotated[AttachmentService, Depends(get_service)], + filename: Annotated[str, Query(min_length=1, max_length=255)], +) -> AttachmentOut: + if len(request.query_params.getlist("filename")) != 1: + raise ApiError(400, "invalid_request", "Неоднозначное имя файла") + return to_output(await service.upload(request, chat_id, user.id, filename)) + + +@router.get("/attachments/{attachment_id}") +async def get_attachment( + attachment_id: UUID, + user: Annotated[UserDTO, Depends(get_current_user)], + service: Annotated[AttachmentService, Depends(get_service)], +) -> AttachmentOut: + return to_output(await service.get(attachment_id, user.id)) + + +@router.get("/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: UUID, + request: Request, + user: Annotated[UserDTO, Depends(get_current_user)], + service: Annotated[AttachmentService, Depends(get_service)], + variant: str = "original", +): + attachment = await service.get(attachment_id, user.id) + if len(request.query_params.getlist("variant")) > 1: + raise ApiError(400, "invalid_request", "Неоднозначный вариант") + if variant not in {"original", "thumbnail", "crop", "optimized"}: + raise ApiError(400, "invalid_request", "Некорректный вариант") + if attachment.status != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + key, mime, size = attachment.object_key, attachment.content_type, attachment.size_bytes + if variant != "original": + item = next((v for v in attachment.variants if v["name"] == variant), None) + if item is None: + raise ApiError(404, "not_found", "Вариант не найден") + key = f"variants/{attachment.id}/v1/{variant}" + mime, size = item["content_type"], item["size_bytes"] + try: + response = await to_thread.run_sync( + lambda: get_storage().get_object(settings.s3_bucket, key) + ) + except STORAGE_ERRORS as error: + raise unavailable(error) from error + + async def content(): + try: + while chunk := await to_thread.run_sync(response.read, 64 * 1024): + yield chunk + finally: + response.close() + response.release_conn() + + return StreamingResponse( + content(), + media_type=mime, + headers={ + "Content-Length": str(size), + }, + ) diff --git a/src/api/v1/routers/auth/__init__.py b/src/api/v1/routers/auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/auth/callback.py b/src/api/v1/routers/auth/callback.py new file mode 100644 index 0000000..45334b8 --- /dev/null +++ b/src/api/v1/routers/auth/callback.py @@ -0,0 +1,31 @@ +import logging + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from src.core.db.repo.auth import fetch_user_credentials +from src.core.security import verify_password_async +from src.core.settings import settings +from src.service.user import UserService + +logger = logging.getLogger(__name__) + + +async def create_admin(session_factory: async_sessionmaker[AsyncSession]) -> None: + async with session_factory.begin() as session: + admin = await UserService(db_session=session).create_admin( + username=settings.admin_username, + display_name=settings.admin_display_name, + password=settings.admin_password.get_secret_value(), + ) + if admin is None and settings.app_env == "production": + credentials = await fetch_user_credentials(session, settings.admin_username) + if credentials is None or not await verify_password_async( + settings.admin_password.get_secret_value(), credentials.password_hash + ): + raise RuntimeError( + "Пароль существующего администратора не совпадает с настройкой production" + ) + if admin is None: + logger.info("Администратор уже существует") + else: + logger.info("Администратор создан: пользователь=%s", admin.id) diff --git a/src/api/v1/routers/auth/router.py b/src/api/v1/routers/auth/router.py new file mode 100644 index 0000000..49076ec --- /dev/null +++ b/src/api/v1/routers/auth/router.py @@ -0,0 +1,50 @@ +from dataclasses import asdict +from typing import Annotated + +from fastapi import APIRouter, Depends, Response +from fastapi.security import HTTPBasicCredentials + +from src.api.dependencies import ( + get_auth_service, + get_basic_credentials, + get_bearer_token, + get_current_user, +) +from src.api.v1.routers.auth.schemas import SessionOut +from src.api.v1.routers.users.schemas import UserOut +from src.core.errors import ApiError +from src.service.auth import AuthService +from src.service.dto import UserDTO +from src.service.exceptions import InvalidCredentialsException + +router = APIRouter(prefix="/auth/sessions", tags=["Сессии"]) + + +@router.post("", status_code=201, response_model=SessionOut) +async def login( + credentials: Annotated[HTTPBasicCredentials, Depends(get_basic_credentials)], + auth: Annotated[AuthService, Depends(get_auth_service)], +) -> SessionOut: + try: + result = await auth.login( + username=credentials.username, + password=credentials.password, + ) + except InvalidCredentialsException as error: + raise ApiError(401, "unauthorized", str(error)) from error + return SessionOut( + token=result.token, + token_type=result.token_type, + expires_at=result.expires_at, + user=UserOut(**asdict(result.user)), + ) + + +@router.delete("/current", status_code=204) +async def logout( + current_user: Annotated[UserDTO, Depends(get_current_user)], + token: Annotated[str, Depends(get_bearer_token)], + auth: Annotated[AuthService, Depends(get_auth_service)], +) -> Response: + await auth.logout(token=token) + return Response(status_code=204) diff --git a/src/api/v1/routers/auth/schemas.py b/src/api/v1/routers/auth/schemas.py new file mode 100644 index 0000000..cdeff56 --- /dev/null +++ b/src/api/v1/routers/auth/schemas.py @@ -0,0 +1,13 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel + +from src.api.v1.routers.users.schemas import UserOut + + +class SessionOut(BaseModel): + token: str + token_type: Literal["Bearer"] + expires_at: datetime + user: UserOut diff --git a/src/api/v1/routers/chats/__init__.py b/src/api/v1/routers/chats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/chats/router.py b/src/api/v1/routers/chats/router.py new file mode 100644 index 0000000..d071839 --- /dev/null +++ b/src/api/v1/routers/chats/router.py @@ -0,0 +1,86 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Query, status + +from src.api.dependencies import get_chat_service, get_current_user +from src.api.v1.routers.chats.schemas import ChatOut, CreateChatIn, GetChatOut +from src.api.v1.schemas import PaginationQuery +from src.core.errors import ApiError +from src.core.pagination import decode_cursor, encode_cursor +from src.service.chat import ChatService +from src.service.dto import ChatDTO, UserDTO +from src.service.exceptions import ChatNotFoundException, InvalidChatMembersException + +router = APIRouter(prefix="/chats", tags=["Чаты"]) + + +def _to_chat_out(chat: ChatDTO) -> ChatOut: + return ChatOut( + id=chat.id, + title=chat.title, + member_ids=list(chat.member_ids), + created_at=chat.created_at, + ) + + +@router.post("", status_code=status.HTTP_201_CREATED) +async def create_chat( + chat_in: CreateChatIn, + current_user: Annotated[UserDTO, Depends(get_current_user)], + chat_service: Annotated[ChatService, Depends(get_chat_service)], +) -> ChatOut: + try: + chat = await chat_service.create_chat( + title=chat_in.title, + member_ids=chat_in.member_ids, + current_user_id=current_user.id, + ) + except InvalidChatMembersException as error: + raise ApiError(400, "invalid_request", str(error)) from error + return _to_chat_out(chat) + + +@router.get("", status_code=status.HTTP_200_OK) +async def get_chats( + pagination: Annotated[PaginationQuery, Query()], + current_user: Annotated[UserDTO, Depends(get_current_user)], + chat_service: Annotated[ChatService, Depends(get_chat_service)], +) -> GetChatOut: + after = None + if pagination.cursor is not None: + after = decode_cursor( + pagination.cursor, + expected_kind="chats", + expected_owner_id=current_user.id, + ) + chats = await chat_service.get_chats( + current_user_id=current_user.id, limit=pagination.limit + 1, after=after + ) + + page = chats[: pagination.limit] + next_cursor = None + if len(chats) > pagination.limit: + last = page[-1] + next_cursor = encode_cursor("chats", current_user.id, last.created_at, last.id) + + return GetChatOut( + items=[_to_chat_out(chat) for chat in page], + next_cursor=next_cursor, + ) + + +@router.get("/{chat_id}", summary="Получить доступный чат") +async def get_chat( + chat_id: UUID, + current_user: Annotated[UserDTO, Depends(get_current_user)], + chat_service: Annotated[ChatService, Depends(get_chat_service)], +) -> ChatOut: + try: + chat = await chat_service.get_chat( + chat_id=chat_id, + current_user_id=current_user.id, + ) + except ChatNotFoundException as error: + raise ApiError(404, "not_found", str(error)) from error + return _to_chat_out(chat) diff --git a/src/api/v1/routers/chats/schemas.py b/src/api/v1/routers/chats/schemas.py new file mode 100644 index 0000000..b51b7f8 --- /dev/null +++ b/src/api/v1/routers/chats/schemas.py @@ -0,0 +1,42 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CreateChatIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_must_not_be_blank(cls, value: str) -> str: + value.encode("utf-8") + if not value.strip(): + raise ValueError("title не может состоять только из пробелов") + return value + + @field_validator("member_ids") + @classmethod + def member_ids_must_be_unique(cls, value: list[UUID]) -> list[UUID]: + if len(value) != len(set(value)): + raise ValueError("member_ids не должен содержать дубликаты") + return value + + +class ChatOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class GetChatOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[ChatOut] + next_cursor: str | None diff --git a/src/api/v1/routers/messages/__init__.py b/src/api/v1/routers/messages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/messages/router.py b/src/api/v1/routers/messages/router.py new file mode 100644 index 0000000..68f9265 --- /dev/null +++ b/src/api/v1/routers/messages/router.py @@ -0,0 +1,96 @@ +import re +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request, status + +from src.api.dependencies import get_current_user, get_message_service +from src.api.v1.routers.messages.schemas import ( + CreateMessageIn, + MessageOut, + MessagePageOut, +) +from src.api.v1.schemas import PaginationQuery +from src.core.errors import ApiError +from src.core.pagination import decode_cursor, encode_cursor +from src.service.dto import MessageDTO, UserDTO +from src.service.exceptions import ChatNotFoundException, IdempotencyConflictException +from src.service.message import MessageService + +router = APIRouter(prefix="/chats", tags=["Сообщения"]) + + +def _to_message_out(message: MessageDTO) -> MessageOut: + return MessageOut( + id=message.id, + chat_id=message.chat_id, + sender_id=message.sender_id, + text=message.text, + attachment_ids=list(message.attachment_ids), + created_at=message.created_at, + ) + + +@router.post("/{chat_id}/messages", status_code=status.HTTP_201_CREATED) +async def create_message( + chat_id: UUID, + message_in: CreateMessageIn, + request: Request, + current_user: Annotated[UserDTO, Depends(get_current_user)], + message_service: Annotated[MessageService, Depends(get_message_service)], + idempotency_key: Annotated[str, Header()], +) -> MessageOut: + if ( + len(request.headers.getlist("idempotency-key")) != 1 + or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) is None + ): + raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key") + try: + message = await message_service.create_message( + chat_id=chat_id, + sender_id=current_user.id, + text=message_in.text, + attachment_ids=message_in.attachment_ids, + idempotency_key=idempotency_key, + ) + except ChatNotFoundException as error: + raise ApiError(404, "not_found", str(error)) from error + except IdempotencyConflictException as error: + raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения") from error + return _to_message_out(message) + + +@router.get("/{chat_id}/messages", status_code=status.HTTP_200_OK) +async def get_messages( + chat_id: UUID, + pagination: Annotated[PaginationQuery, Query()], + current_user: Annotated[UserDTO, Depends(get_current_user)], + message_service: Annotated[MessageService, Depends(get_message_service)], +) -> MessagePageOut: + after = None + if pagination.cursor is not None: + after = decode_cursor( + pagination.cursor, + expected_kind="messages", + expected_owner_id=chat_id, + ) + try: + messages = await message_service.get_messages( + chat_id=chat_id, + current_user_id=current_user.id, + limit=pagination.limit + 1, + after=after, + ) + except ChatNotFoundException as error: + raise ApiError(404, "not_found", str(error)) from error + + page = messages[: pagination.limit] + next_cursor = None + if len(messages) > pagination.limit: + last = page[-1] + next_cursor = encode_cursor("messages", chat_id, last.created_at, last.id) + + return MessagePageOut( + items=[_to_message_out(message) for message in page], + next_cursor=next_cursor, + ) diff --git a/src/api/v1/routers/messages/schemas.py b/src/api/v1/routers/messages/schemas.py new file mode 100644 index 0000000..1a0c821 --- /dev/null +++ b/src/api/v1/routers/messages/schemas.py @@ -0,0 +1,43 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class CreateMessageIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def text_must_not_be_blank(cls, value: str) -> str: + value.encode("utf-8") + return value + + @model_validator(mode="after") + def validate_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Требуется текст или вложение") + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Вложения не должны повторяться") + return self + + +class MessageOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + sender_id: UUID + text: str + attachment_ids: list[UUID] + created_at: datetime + + +class MessagePageOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[MessageOut] + next_cursor: str | None diff --git a/src/api/v1/routers/users/__init__.py b/src/api/v1/routers/users/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/users/router.py b/src/api/v1/routers/users/router.py new file mode 100644 index 0000000..cbe3726 --- /dev/null +++ b/src/api/v1/routers/users/router.py @@ -0,0 +1,44 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends, status + +from src.api.dependencies import get_current_user, get_user_service +from src.api.v1.routers.users.schemas import CreateUserIn, UserOut +from src.core.errors import ApiError +from src.service.dto import UserDTO +from src.service.exceptions import UserAlreadyExistsException +from src.service.user import UserService + +router = APIRouter(prefix="/users", tags=["Пользователи"]) + + +def _to_user_out(user: UserDTO) -> UserOut: + return UserOut( + id=user.id, + username=user.username, + display_name=user.display_name, + created_at=user.created_at, + ) + + +@router.post("", status_code=status.HTTP_201_CREATED) +async def create_user( + user_in: CreateUserIn, + user_service: Annotated[UserService, Depends(get_user_service)], +) -> UserOut: + try: + user = await user_service.create_user( + password=user_in.password, + username=user_in.username, + display_name=user_in.display_name, + ) + except UserAlreadyExistsException as error: + raise ApiError(409, "conflict", str(error)) from error + return _to_user_out(user) + + +@router.get("/me", status_code=status.HTTP_200_OK) +async def get_me( + current_user: Annotated[UserDTO, Depends(get_current_user)], +) -> UserOut: + return _to_user_out(current_user) diff --git a/src/api/v1/routers/users/schemas.py b/src/api/v1/routers/users/schemas.py new file mode 100644 index 0000000..7cac9da --- /dev/null +++ b/src/api/v1/routers/users/schemas.py @@ -0,0 +1,29 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CreateUserIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + password: str = Field(min_length=12, max_length=128) + display_name: str = Field(min_length=1, max_length=100) + + @field_validator("display_name", "password") + @classmethod + def required_string_must_not_be_blank(cls, value: str) -> str: + value.encode("utf-8") + if not value.strip(): + raise ValueError("Обязательная строка не может состоять только из пробелов") + return value + + +class UserOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + username: str + display_name: str + created_at: datetime diff --git a/src/api/v1/schemas.py b/src/api/v1/schemas.py new file mode 100644 index 0000000..6310689 --- /dev/null +++ b/src/api/v1/schemas.py @@ -0,0 +1,21 @@ +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from src.core.settings import settings + + +class PaginationQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + limit: int = Field(default=50, ge=1, le=100) + cursor: str | None = Field(default=None, min_length=1, max_length=settings.cursor_max_length) + + @field_validator("limit", mode="before") + @classmethod + def limit_must_be_integer(cls, value): + if isinstance(value, bool) or isinstance(value, float): + raise ValueError("limit должен быть целым числом") + if isinstance(value, str) and re.fullmatch(r"[0-9]+", value) is None: + raise ValueError("limit должен содержать десятичные цифры") + return value diff --git a/src/core/__init__.py b/src/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/cache.py b/src/core/cache.py new file mode 100644 index 0000000..19444bf --- /dev/null +++ b/src/core/cache.py @@ -0,0 +1,68 @@ +"""Клиент Redis и необязательный кэш профилей пользователей.""" + +import logging + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from src.core.settings import settings + +logger = logging.getLogger(__name__) +cache: Redis | None = None + + +def open_cache(): + global cache + cache = Redis.from_url( + settings.redis_url.get_secret_value(), + decode_responses=True, + max_connections=settings.redis_max_connections, + socket_connect_timeout=settings.redis_connect_timeout_seconds, + socket_timeout=settings.redis_read_timeout_seconds, + retry_on_timeout=settings.redis_retry_on_timeout, + ) + + +async def close_cache(): + global cache + if cache is not None: + await cache.aclose() + cache = None + + +async def get(key: str) -> str | None: + if cache is None: + return None + try: + return await cache.get(key) + except UnicodeError: + logger.warning("Повреждённый UTF-8 в кэше профиля; читаем PostgreSQL") + await delete(key) + return None + except (RedisError, OSError): + logger.warning("Redis недоступен; читаем профиль из PostgreSQL") + return None + + +async def set(key: str, value: str): + if cache is None: + return + try: + await cache.set(key, value, ex=settings.profile_cache_ttl_seconds) + except (RedisError, OSError): + logger.warning("Redis недоступен; пропускаем запись кэша") + + +async def delete(key: str) -> None: + if cache is None: + return + try: + await cache.delete(key) + except (RedisError, OSError): + logger.warning("Redis недоступен; пропускаем удаление кэша профиля") + + +def get_client() -> Redis: + if cache is None: + raise RedisError("Клиент Redis не настроен") + return cache diff --git a/src/core/callback.py b/src/core/callback.py new file mode 100644 index 0000000..3cf0cff --- /dev/null +++ b/src/core/callback.py @@ -0,0 +1,24 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from src.api.v1.routers.auth.callback import create_admin +from src.core import readiness +from src.core.db import session as database + +STARTUP_CALLBACKS = (create_admin,) + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + async with database.database_lifespan(app): + session_factory = database.session_factory + if session_factory is None: + raise RuntimeError("Фабрика сессий не запущена") + for callback in STARTUP_CALLBACKS: + await callback(session_factory) + try: + yield + finally: + await readiness.close() diff --git a/src/core/db/__init__.py b/src/core/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/db/models.py b/src/core/db/models.py new file mode 100644 index 0000000..2b57a4e --- /dev/null +++ b/src/core/db/models.py @@ -0,0 +1,207 @@ +from datetime import datetime +from uuid import UUID + +from sqlalchemy import ( + JSON, + BigInteger, + Boolean, + CheckConstraint, + DateTime, + ForeignKey, + ForeignKeyConstraint, + Index, + Integer, + String, + Text, + UniqueConstraint, + false, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from src.core.db.types import Utf8Text + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + __table_args__ = ( + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + CheckConstraint( + "octet_length(display_name) BETWEEN 1 AND 400", name="users_display_name_bytes_check" + ), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(String(32), unique=True) + display_name: Mapped[str] = mapped_column(Utf8Text()) + password_hash: Mapped[str] = mapped_column(Text) + is_admin: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class Chat(Base): + __tablename__ = "chats" + __table_args__ = ( + CheckConstraint("octet_length(title) BETWEEN 1 AND 400", name="chats_title_bytes_check"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + title: Mapped[str] = mapped_column(Utf8Text()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + members: Mapped[list["ChatMember"]] = relationship(order_by="ChatMember.user_id") + + +class ChatMember(Base): + __tablename__ = "chat_members" + __table_args__ = (Index("chat_members_user_idx", "user_id", "chat_id"),) + + chat_id: Mapped[UUID] = mapped_column( + ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True + ) + user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + + +class Message(Base): + __tablename__ = "messages" + __table_args__ = ( + ForeignKeyConstraint( + ["chat_id", "sender_id"], ["chat_members.chat_id", "chat_members.user_id"] + ), + CheckConstraint("octet_length(text) BETWEEN 0 AND 16000", name="messages_text_bytes_check"), + UniqueConstraint("chat_id", "id"), + Index("messages_history_idx", "chat_id", "created_at", "id"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey("chats.id")) + sender_id: Mapped[UUID] = mapped_column(ForeignKey("users.id")) + attachments: Mapped[list["MessageAttachment"]] = relationship( + lazy="selectin", order_by="MessageAttachment.position" + ) + text: Mapped[str] = mapped_column(Utf8Text()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class MessageRequest(Base): + # Таблица обслуживает только POST /chats/{chat_id}/messages: метод и путь + # однозначно заданы таблицей и chat_id. Общая область ключа: sender/chat/key. + __tablename__ = "message_requests" + + sender_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey("chats.id"), primary_key=True) + key: Mapped[str] = mapped_column(String(128), primary_key=True) + request_text: Mapped[str] = mapped_column(Utf8Text()) + request_attachment_ids: Mapped[str] = mapped_column(Text, default="", server_default="") + message_id: Mapped[UUID | None] = mapped_column(ForeignKey("messages.id")) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class Attachment(Base): + __tablename__ = "attachments" + __table_args__ = ( + UniqueConstraint("chat_id", "id"), + CheckConstraint("size_bytes BETWEEN 1 AND 10485760"), + CheckConstraint( + "status IN ('queued', 'processing', 'ready', 'failed')", name="attachments_status_check" + ), + CheckConstraint("content_type IN ('image/png', 'image/jpeg')"), + Index("attachments_chat_idx", "chat_id"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey("chats.id")) + owner_id: Mapped[UUID] = mapped_column(ForeignKey("users.id")) + filename: Mapped[str] = mapped_column(String(255)) + object_key: Mapped[str] = mapped_column(Text, unique=True) + content_type: Mapped[str] = mapped_column(String(32)) + size_bytes: Mapped[int] = mapped_column(BigInteger) + sha256: Mapped[str] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + storage_missing_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + status: Mapped[str] = mapped_column(String(16), default="ready", server_default="ready") + error_code: Mapped[str | None] = mapped_column(String(64)) + staging_key: Mapped[str | None] = mapped_column(Text) + variants: Mapped[list] = mapped_column(JSON, default=list, server_default="[]") + + +class AttachmentJob(Base): + """A job is also its transactional outbox: published_at=NULL means send it.""" + + __tablename__ = "attachment_jobs" + __table_args__ = ( + UniqueConstraint("attachment_id", "pipeline_version"), + CheckConstraint("state IN ('queued', 'processing', 'ready', 'failed')"), + CheckConstraint("attempt >= 1 AND cycle_attempt >= 1"), + CheckConstraint("pipeline_version = 1"), + Index("attachment_jobs_dispatch_idx", "state", "next_run_at"), + ) + id: Mapped[UUID] = mapped_column(primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(ForeignKey("attachments.id")) + pipeline_version: Mapped[int] = mapped_column(Integer, default=1) + state: Mapped[str] = mapped_column(String(16), default="queued") + attempt: Mapped[int] = mapped_column(Integer, default=1) + cycle_attempt: Mapped[int] = mapped_column(Integer, default=1) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + next_run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + claim_token: Mapped[UUID | None] = mapped_column() + error_code: Mapped[str | None] = mapped_column(String(64)) + traceparent: Mapped[str | None] = mapped_column(String(128)) + tracestate: Mapped[str | None] = mapped_column(String(512)) + fault: Mapped[str | None] = mapped_column(String(32)) + + +class JobReplay(Base): + __tablename__ = "job_replays" + id: Mapped[UUID] = mapped_column(primary_key=True) + job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id")) + operator: Mapped[str] = mapped_column(String(100)) + reason: Mapped[str] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + attempt: Mapped[int] = mapped_column(Integer) + + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + __table_args__ = ( + ForeignKeyConstraint( + ["chat_id", "message_id"], ["messages.chat_id", "messages.id"], ondelete="CASCADE" + ), + ForeignKeyConstraint( + ["chat_id", "attachment_id"], ["attachments.chat_id", "attachments.id"] + ), + UniqueConstraint("message_id", "position"), + CheckConstraint("position BETWEEN 0 AND 9"), + ) + + message_id: Mapped[UUID] = mapped_column(primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] = mapped_column() + position: Mapped[int] = mapped_column(Integer) + + +class UploadIntent(Base): + __tablename__ = "upload_intents" + __table_args__ = (CheckConstraint("state IN ('pending', 'committed', 'abandoned')"),) + + object_key: Mapped[str] = mapped_column(Text, primary_key=True) + state: Mapped[str] = mapped_column(String(16)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class StorageFinding(Base): + __tablename__ = "storage_findings" + __table_args__ = (CheckConstraint("kind IN ('orphan', 'missing')"),) + + object_key: Mapped[str] = mapped_column(Text, primary_key=True) + kind: Mapped[str] = mapped_column(String(16), primary_key=True) + fingerprint: Mapped[str | None] = mapped_column(Text) + first_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + last_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) diff --git a/src/core/db/repo/__init__.py b/src/core/db/repo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/db/repo/auth.py b/src/core/db/repo/auth.py new file mode 100644 index 0000000..2497013 --- /dev/null +++ b/src/core/db/repo/auth.py @@ -0,0 +1,16 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.models import User +from src.core.db.repo.user import to_user +from src.service.dto import UserCredentialsDTO + + +async def fetch_user_credentials( + session: AsyncSession, + username: str, +) -> UserCredentialsDTO | None: + user = await session.scalar(select(User).where(User.username == username)) + if user is None: + return None + return UserCredentialsDTO(user=to_user(user), password_hash=user.password_hash) diff --git a/src/core/db/repo/chat.py b/src/core/db/repo/chat.py new file mode 100644 index 0000000..aa88770 --- /dev/null +++ b/src/core/db/repo/chat.py @@ -0,0 +1,59 @@ +from datetime import datetime +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from src.core.db.models import Chat, ChatMember +from src.service.dto import ChatDTO + + +def to_chat(chat: Chat) -> ChatDTO: + return ChatDTO( + chat.id, chat.title, tuple(member.user_id for member in chat.members), chat.created_at + ) + + +async def fetch_chat_by_id(session: AsyncSession, chat_id: UUID) -> ChatDTO | None: + chat = await session.scalar( + select(Chat).options(selectinload(Chat.members)).where(Chat.id == chat_id), + ) + return to_chat(chat) if chat else None + + +async def fetch_chats( + session: AsyncSession, + user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, +) -> list[ChatDTO]: + query = ( + select(Chat) + .options(selectinload(Chat.members)) + .join(ChatMember) + .where(ChatMember.user_id == user_id) + ) + if after is not None: + query = query.where(tuple_(Chat.created_at, Chat.id) > after) + chats = await session.scalars(query.order_by(Chat.created_at, Chat.id).limit(limit)) + return [to_chat(chat) for chat in chats] + + +async def create_chat( + session: AsyncSession, + chat_id: UUID, + title: str, + member_ids: list[UUID], + created_at: datetime, +) -> ChatDTO: + chat = Chat( + id=chat_id, + title=title, + created_at=created_at, + members=[ChatMember(user_id=user_id) for user_id in member_ids], + ) + session.add(chat) + # ORM сохраняет чат и участников в транзакции текущего запроса. + await session.flush() + return to_chat(chat) diff --git a/src/core/db/repo/message.py b/src/core/db/repo/message.py new file mode 100644 index 0000000..35036a6 --- /dev/null +++ b/src/core/db/repo/message.py @@ -0,0 +1,151 @@ +from datetime import datetime +from uuid import UUID + +from sqlalchemy import or_, select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.models import Attachment, Message, MessageAttachment, MessageRequest +from src.service.dto import MessageDTO + + +def to_message(message: Message) -> MessageDTO: + return MessageDTO( + message.id, + message.chat_id, + message.sender_id, + message.text, + message.created_at, + tuple(link.attachment_id for link in message.attachments), + ) + + +def history_query(chat_id: UUID, limit: int, after: tuple[datetime, UUID] | None = None): + query = select(Message).where(Message.chat_id == chat_id) + if after is not None: + query = query.where(tuple_(Message.created_at, Message.id) > after) + return query.order_by(Message.created_at, Message.id).limit(limit) + + +async def fetch_messages_by_chat_id( + session: AsyncSession, + chat_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, +) -> list[MessageDTO]: + messages = await session.scalars(history_query(chat_id, limit, after)) + return [to_message(message) for message in messages] + + +async def create_message( + session: AsyncSession, + message_id: UUID, + chat_id: UUID, + sender_id: UUID, + text: str, + created_at: datetime, + attachment_ids: list[UUID] | None = None, +) -> MessageDTO: + message = Message( + id=message_id, + chat_id=chat_id, + sender_id=sender_id, + text=text, + created_at=created_at, + attachments=[ + MessageAttachment(chat_id=chat_id, attachment_id=aid, position=i) + for i, aid in enumerate(attachment_ids or []) + ], + ) + session.add(message) + await session.flush() + return to_message(message) + + +async def create_idempotent_message( + session: AsyncSession, + message_id: UUID, + chat_id: UUID, + sender_id: UUID, + text: str, + created_at: datetime, + idempotency_key: str, + attachment_ids: list[UUID] | None = None, +) -> MessageDTO: + from sqlalchemy import func, update + from sqlalchemy import text as sql_text + from sqlalchemy.dialects.postgresql import insert + + from src.core.errors import ApiError + from src.service.exceptions import IdempotencyConflictException + + attachment_ids = attachment_ids or [] + request_attachments = ",".join(str(aid) for aid in attachment_ids) + if attachment_ids: + found = set( + await session.scalars( + select(Attachment.id).where( + Attachment.chat_id == chat_id, Attachment.id.in_(attachment_ids) + ) + ) + ) + if found != set(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено") + + identity = ( + MessageRequest.sender_id == sender_id, + MessageRequest.chat_id == chat_id, + MessageRequest.key == idempotency_key, + ) + # UNIQUE/UPSERT сериализует конкурентные повторы даже на разных процессах. + # Проигравший ждёт COMMIT; при rollback он сам получает право на создание. + values = dict( + sender_id=sender_id, + chat_id=chat_id, + key=idempotency_key, + request_text=text, + request_attachment_ids=request_attachments, + message_id=None, + expires_at=func.clock_timestamp() + sql_text("interval '24 hours'"), + ) + claim = ( + insert(MessageRequest) + .values(**values) + .on_conflict_do_update( + index_elements=["sender_id", "chat_id", "key"], + set_={ + k: v + for k, v in values.items() + if k in {"request_text", "request_attachment_ids", "message_id", "expires_at"} + }, + where=MessageRequest.expires_at <= func.clock_timestamp(), + ) + .returning(MessageRequest.key) + ) + owned = await session.scalar(claim) + if owned is None: + existing = (await session.execute(select(MessageRequest).where(*identity))).scalar_one() + if existing.request_text != text or existing.request_attachment_ids != request_attachments: + raise IdempotencyConflictException + message = await session.get(Message, existing.message_id) + if message is None: + raise RuntimeError("Нарушена атомарность результата идемпотентности") + return to_message(message) + + if attachment_ids: + missing = await session.scalar( + select(Attachment.id) + .where( + Attachment.id.in_(attachment_ids), + or_(Attachment.storage_missing_since.is_not(None), Attachment.status == "failed"), + ) + .limit(1) + ) + if missing is not None: + raise ApiError(409, "conflict", "Оригинал вложения недоступен") + + message = await create_message( + session, message_id, chat_id, sender_id, text, created_at, attachment_ids + ) + await session.execute(update(MessageRequest).where(*identity).values(message_id=message.id)) + # COMMIT сообщения и ключа выполняется вместе в get_db_session до HTTP 201. + return message diff --git a/src/core/db/repo/user.py b/src/core/db/repo/user.py new file mode 100644 index 0000000..6f10eed --- /dev/null +++ b/src/core/db/repo/user.py @@ -0,0 +1,120 @@ +import json +import re +from dataclasses import asdict +from datetime import datetime, timezone +from uuid import UUID + +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core import cache +from src.core.db.models import User +from src.service.dto import UserDTO + + +def to_user(user: User) -> UserDTO: + return UserDTO(user.id, user.username, user.display_name, user.created_at) + + +async def fetch_user_by_id(session: AsyncSession, user_id: UUID) -> UserDTO | None: + key = f"user:v1:{user_id}" + cached = await cache.get(key) + if cached is not None: + try: + data = json.loads(cached) + if ( + not isinstance(data, dict) + or set(data) != {"id", "username", "display_name", "created_at"} + or not all(isinstance(value, str) for value in data.values()) + ): + raise ValueError + created_at = datetime.fromisoformat(data["created_at"]) + if created_at.tzinfo is None: + raise ValueError + data["display_name"].encode("utf-8") + user = UserDTO( + UUID(data["id"]), + data["username"], + data["display_name"], + created_at.astimezone(timezone.utc), + ) + if ( + user.id != user_id + or re.fullmatch(r"[a-z0-9_]{3,32}", user.username) is None + or not 1 <= len(user.display_name) <= 100 + or not user.display_name.strip() + or user.created_at.tzinfo is None + ): + raise ValueError + return user + except (ValueError, KeyError, TypeError, OverflowError): + await cache.delete(key) + model = await session.get(User, user_id) + user = to_user(model) if model else None + if user: + await cache.set(key, json.dumps(asdict(user), default=str)) + return user + + +async def fetch_user_by_username(session: AsyncSession, username: str) -> UserDTO | None: + model = await session.scalar(select(User).where(User.username == username)) + return to_user(model) if model else None + + +async def create_user( + session: AsyncSession, + user_id: UUID, + username: str, + display_name: str, + created_at: datetime, + password_hash: str, +) -> UserDTO: + user = User( + id=user_id, + username=username, + display_name=display_name, + created_at=created_at, + password_hash=password_hash, + ) + session.add(user) + await session.flush() + return to_user(user) + + +async def users_exist(session: AsyncSession, user_ids: list[UUID]) -> bool: + count = await session.scalar( + select(func.count()).select_from(User).where(User.id.in_(user_ids)) + ) + return count == len(user_ids) + + +async def create_admin( + session: AsyncSession, + user_id: UUID, + username: str, + display_name: str, + password_hash: str, + created_at: datetime, +) -> UserDTO | None: + user = await session.scalar( + insert(User) + .values( + id=user_id, + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=created_at, + is_admin=True, + ) + .on_conflict_do_nothing(index_elements=[User.username]) + .returning(User) + ) + return to_user(user) if user else None + + +async def admin_exists(session: AsyncSession, username: str) -> bool: + user_id = await session.scalar( + select(User.id).where(User.username == username, User.is_admin.is_(True)) + ) + return user_id is not None diff --git a/src/core/db/session.py b/src/core/db/session.py new file mode 100644 index 0000000..ef34790 --- /dev/null +++ b/src/core/db/session.py @@ -0,0 +1,105 @@ +from contextlib import asynccontextmanager + +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import AsyncAdaptedQueuePool + +from src.core.cache import close_cache, open_cache +from src.core.settings import settings +from src.observability import signals + +session_factory: async_sessionmaker[AsyncSession] | None = None + + +class ObservedPool(AsyncAdaptedQueuePool): + waiting = 0 + + def _do_get(self): + self.waiting += 1 + try: + return super()._do_get() + finally: + self.waiting -= 1 + + +def create_engine(): + engine = create_async_engine( + settings.sqlalchemy_database_url, + poolclass=ObservedPool, + pool_size=settings.database_pool_size, + max_overflow=settings.database_max_overflow, + pool_timeout=settings.database_pool_timeout_seconds, + pool_pre_ping=settings.database_pool_pre_ping, + hide_parameters=True, + connect_args={ + "timeout": settings.database_connect_timeout_seconds, + "command_timeout": settings.database_command_timeout_seconds, + "server_settings": {"statement_timeout": str(settings.database_statement_timeout_ms)}, + }, + ) + + signals.register_pool(engine) + + @event.listens_for(engine.sync_engine, "before_cursor_execute") + def before(conn, cursor, statement, parameters, context, executemany): + operation = statement.lstrip().split(None, 1)[0].upper() if statement.strip() else "OTHER" + if operation not in {"SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK"}: + operation = "OTHER" + span = signals.tracer.start_span( + "db." + operation, + kind=SpanKind.CLIENT, + attributes={"db.system": "postgresql", "db.operation.name": operation}, + ) + scope = trace.use_span( + span, end_on_exit=True, record_exception=False, set_status_on_exception=False + ) + scope.__enter__() + context._telemetry_scope = scope + + @event.listens_for(engine.sync_engine, "after_cursor_execute") + def after(conn, cursor, statement, parameters, context, executemany): + scope = getattr(context, "_telemetry_scope", None) + if scope: + scope.__exit__(None, None, None) + context._telemetry_scope = None + + @event.listens_for(engine.sync_engine, "handle_error") + def failed(exception_context): + context = exception_context.execution_context + scope = getattr(context, "_telemetry_scope", None) + if scope: + trace.get_current_span().set_status(Status(StatusCode.ERROR)) + trace.get_current_span().set_attribute( + "error.type", type(exception_context.original_exception).__name__ + ) + scope.__exit__(None, None, None) + context._telemetry_scope = None + signals.event("db.error", error_type=type(exception_context.original_exception).__name__) + + return engine + + +@asynccontextmanager +async def database_lifespan(app): + global session_factory + engine = create_engine() + session_factory = async_sessionmaker(engine, expire_on_commit=False) + open_cache() + try: + yield + finally: + await close_cache() + await engine.dispose() + session_factory = None + + +async def get_db_session(): + if session_factory is None: + raise RuntimeError("Фабрика сессий не запущена") + async with session_factory() as session: + # Одна транзакция на запрос: аутентификация и бизнес-операция вместе. + # Depends(scope='function') завершает COMMIT до отправки ответа. + async with session.begin(): + yield session diff --git a/src/core/db/types.py b/src/core/db/types.py new file mode 100644 index 0000000..36501fd --- /dev/null +++ b/src/core/db/types.py @@ -0,0 +1,15 @@ +"""Unicode-строки в UTF-8 bytea: PostgreSQL text не допускает U+0000.""" + +from sqlalchemy import LargeBinary +from sqlalchemy.types import TypeDecorator + + +class Utf8Text(TypeDecorator[str]): + impl = LargeBinary + cache_ok = True + + def process_bind_param(self, value: str | None, dialect) -> bytes | None: + return None if value is None else value.encode("utf-8") + + def process_result_value(self, value: bytes | None, dialect) -> str | None: + return None if value is None else value.decode("utf-8") diff --git a/src/core/errors.py b/src/core/errors.py new file mode 100644 index 0000000..70a62c9 --- /dev/null +++ b/src/core/errors.py @@ -0,0 +1,29 @@ +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str) -> None: + self.status_code = status_code + self.code = code + self.message = message + super().__init__(message) + + +def error_response( + request: Request, + status_code: int, + code: str, + message: str, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", "неизвестен") + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + ) diff --git a/src/core/pagination.py b/src/core/pagination.py new file mode 100644 index 0000000..d49e7b7 --- /dev/null +++ b/src/core/pagination.py @@ -0,0 +1,43 @@ +import base64 +import json +from datetime import datetime, timezone +from uuid import UUID + +from src.core.errors import ApiError + + +def encode_cursor(kind: str, owner_id: UUID, created_at: datetime, item_id: UUID) -> str: + payload = { + "kind": kind, + "owner_id": str(owner_id), # id чата + "created_at": created_at.isoformat(), # его время + "item_id": str(item_id), # id сообщения + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor( + cursor: str, expected_kind: str, expected_owner_id: UUID +) -> tuple[datetime, UUID]: + try: + padding = "=" * (-len(cursor) % 4) + raw = base64.b64decode(cursor + padding, altchars=b"-_", validate=True) + payload = json.loads(raw.decode("utf-8")) + if ( + not isinstance(payload, dict) + or set(payload) != {"kind", "owner_id", "created_at", "item_id"} + or not all(isinstance(value, str) for value in payload.values()) + ): + raise ValueError + if payload["kind"] != expected_kind: + raise ValueError + if UUID(payload["owner_id"]) != expected_owner_id: + raise ValueError + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + if created_at.tzinfo is None: + raise ValueError + return created_at.astimezone(timezone.utc), item_id + except (KeyError, TypeError, ValueError, UnicodeError, OverflowError): + raise ApiError(400, "invalid_request", "Некорректный курсор") from None diff --git a/src/core/readiness.py b/src/core/readiness.py new file mode 100644 index 0000000..133c3db --- /dev/null +++ b/src/core/readiness.py @@ -0,0 +1,89 @@ +"""Ограниченная проверка обязательного пути, включая повреждение схемы. + +Отмена asyncpg может ждать ответа замороженного PostgreSQL. HTTP readiness +не должна ждать завершения такого rollback; незавершённая проба остаётся одной +на процесс и до её завершения сервер считается неготовым. +""" + +import asyncio + +from anyio import to_thread +from sqlalchemy import select + +from src.core import cache +from src.core.db import session as database +from src.core.db.models import ( + Attachment, + Chat, + ChatMember, + Message, + MessageAttachment, + MessageRequest, + StorageFinding, + UploadIntent, + User, +) +from src.core.errors import ApiError +from src.core.settings import settings +from src.core.storage import STORAGE_ERRORS, get_storage, unavailable + +_pending: asyncio.Task[None] | None = None + + +async def _probe() -> None: + factory = database.session_factory + if factory is None: + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + async with factory() as session: + for model in ( + User, + Chat, + ChatMember, + Message, + MessageRequest, + Attachment, + StorageFinding, + UploadIntent, + MessageAttachment, + ): + await session.execute(select(model).limit(0)) + await cache.get_client().ping() + try: + exists = await to_thread.run_sync( + lambda: get_storage().bucket_exists(settings.s3_bucket) + ) + except STORAGE_ERRORS as error: + raise unavailable(error) from error + if not exists: + raise ApiError(503, "unavailable", "Хранилище файлов не готово") + + +def _consume_result(task: asyncio.Task[None]) -> None: + if not task.cancelled(): + task.exception() + + +async def check() -> None: + global _pending + if _pending is None or _pending.done(): + _pending = asyncio.create_task(_probe()) + _pending.add_done_callback(_consume_result) + task = _pending + # wait(), в отличие от wait_for(), не ждёт завершения отмены драйвером. + done, _ = await asyncio.wait({task}, timeout=settings.healthcheck_timeout_seconds) + if not done: + if not task.cancelling(): + task.cancel() + raise ApiError(503, "unavailable", "Проверка готовности превысила время ожидания") + if task.cancelled(): + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + task.result() + + +async def close() -> None: + global _pending + if _pending is not None and not _pending.done(): + if not _pending.cancelling(): + _pending.cancel() + await asyncio.wait({_pending}, timeout=settings.healthcheck_timeout_seconds) + _pending = None diff --git a/src/core/request_deadline.py b/src/core/request_deadline.py new file mode 100644 index 0000000..b79939a --- /dev/null +++ b/src/core/request_deadline.py @@ -0,0 +1,77 @@ +"""Bound the time to response headers, including slow DB cancellation/rollback. + +A timed-out handler is cancelled but not awaited: asyncpg can wait for a frozen +server during rollback. Such handlers retain their slot until they actually end. +This prevents an outage from accumulating unbounded background handlers. +""" + +import asyncio +import logging +import re + +from starlette.requests import Request + +from src.core.errors import error_response +from src.core.settings import settings + +logger = logging.getLogger(__name__) + + +class RequestDeadlineMiddleware: + def __init__(self, app): + self.app = app + self.pending = set() + + def completed(self, task): + self.pending.discard(task) + if not task.cancelled(): + error = task.exception() + if error is not None: + logger.warning("Обработчик запроса завершился с ошибкой: %s", type(error).__name__) + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or not scope["path"].startswith("/api/v1/"): + await self.app(scope, receive, send) + return + if len(self.pending) >= settings.http_concurrency_limit: + await error_response(Request(scope), 503, "unavailable", "Сервис перегружен")( + scope, receive, send + ) + return + started = asyncio.Event() + expired = False + + async def guarded_send(message): + if expired: + return + if message["type"] == "http.response.start": + started.set() + await send(message) + + task = asyncio.create_task(self.app(scope, receive, guarded_send)) + self.pending.add(task) + task.add_done_callback(self.completed) + start_waiter = asyncio.create_task(started.wait()) + timeout = settings.http_request_timeout_seconds + if scope["method"] == "POST" and re.fullmatch( + r"/api/v1/chats/[^/]+/attachments/?", scope["path"] + ): + # File transfer has its own byte/time bounds; allow its receive and + # S3 stages before applying the final response-header deadline. + timeout += settings.upload_timeout_seconds + 2 * settings.s3_timeout_seconds + try: + done, _ = await asyncio.wait({task, start_waiter}, timeout=timeout) + if done: + await task + else: + expired = True + task.cancel() + await error_response( + Request(scope), 503, "unavailable", "Истекло время обработки запроса" + )(scope, receive, send) + except asyncio.CancelledError: + expired = True + task.cancel() + raise + finally: + start_waiter.cancel() diff --git a/src/core/request_limits.py b/src/core/request_limits.py new file mode 100644 index 0000000..ddf7b63 --- /dev/null +++ b/src/core/request_limits.py @@ -0,0 +1,132 @@ +"""Ограниченный приём тела до валидации FastAPI, включая chunked-запросы.""" + +import asyncio +import json +import re + +from anyio import CapacityLimiter, WouldBlock +from starlette.requests import Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from src.core.errors import error_response +from src.core.settings import settings + + +def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Повторяющийся ключ JSON") + result[key] = value + return result + + +def reject_constant(value: str) -> None: + raise ValueError("Недопустимое значение JSON") + + +class RequestLimitsMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + self.limiter = CapacityLimiter(settings.http_concurrency_limit) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or not scope["path"].startswith("/api/v1/"): + await self.app(scope, receive, send) + return + try: + self.limiter.acquire_nowait() + except WouldBlock: + await error_response(Request(scope), 503, "unavailable", "Сервис перегружен")( + scope, receive, send + ) + return + try: + await self.handle_request(scope, receive, send) + finally: + self.limiter.release() + + async def handle_request(self, scope: Scope, receive: Receive, send: Send) -> None: + request = Request(scope) + if len(request.headers.getlist("content-type")) > 1 or any( + len(request.query_params.getlist(name)) > 1 for name in ("limit", "cursor") + ): + await error_response( + request, 400, "invalid_request", "Неоднозначные параметры запроса" + )(scope, receive, send) + return + # Binary uploads enforce their own streaming limit after authentication. + # Do not buffer them or apply the JSON-only 64 KiB limit. + upload = ( + request.method == "POST" + and re.fullmatch(r"/api/v1/chats/[^/]+/attachments/?", scope["path"]) is not None + ) + if upload: + await self.app(scope, receive, send) + return + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + json_body = ( + request.method == "POST" and scope["path"].rstrip("/") != "/api/v1/auth/sessions" + ) + if json_body and content_type != "application/json": + await error_response( + request, 415, "unsupported_media_type", "Требуется application/json" + )(scope, receive, send) + return + try: + length = int(request.headers.get("content-length", "0")) + if length < 0: + raise ValueError + except ValueError: + await error_response(request, 400, "invalid_request", "Неверная длина тела")( + scope, receive, send + ) + return + body = bytearray() + too_large = length > settings.http_max_body_bytes + if not too_large: + try: + async with asyncio.timeout(settings.http_body_timeout_seconds): + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + chunk = message.get("body", b"") + if len(body) + len(chunk) > settings.http_max_body_bytes: + too_large = True + break + body.extend(chunk) + if not message.get("more_body", False): + break + except TimeoutError: + await error_response(request, 408, "invalid_request", "Истекло время приёма тела")( + scope, receive, send + ) + return + if too_large: + await error_response(request, 413, "payload_too_large", "Превышен размер тела запроса")( + scope, receive, send + ) + return + if json_body: + try: + json.loads( + body.decode("utf-8"), + object_pairs_hook=unique_object, + parse_constant=reject_constant, + ) + except (ValueError, UnicodeError, RecursionError): + await error_response(request, 400, "invalid_request", "Некорректный JSON UTF-8")( + scope, receive, send + ) + return + delivered = False + + async def buffered_receive() -> Message: + nonlocal delivered + if delivered: + return await receive() + delivered = True + return {"type": "http.request", "body": bytes(body), "more_body": False} + + await self.app(scope, buffered_receive, send) diff --git a/src/core/security.py b/src/core/security.py new file mode 100644 index 0000000..210710c --- /dev/null +++ b/src/core/security.py @@ -0,0 +1,42 @@ +"""Пароли: Argon2id. Токены: SHA-256 от случайного секрета.""" + +import hashlib + +from anyio import CapacityLimiter, to_thread +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError + +from src.core.settings import settings + +password_hasher = PasswordHasher( + time_cost=settings.argon2_time_cost, + memory_cost=settings.argon2_memory_cost, + parallelism=settings.argon2_parallelism, + salt_len=settings.argon2_salt_len, + hash_len=settings.argon2_hash_len, + type=Type.ID, +) +password_limiter = CapacityLimiter(settings.password_hash_concurrency) + + +async def hash_password_async(password: str) -> str: + return await to_thread.run_sync(hash_password, password, limiter=password_limiter) + + +async def verify_password_async(password: str, stored: str) -> bool: + return await to_thread.run_sync(verify_password, password, stored, limiter=password_limiter) + + +def hash_password(password: str) -> str: + return password_hasher.hash(password) + + +def verify_password(password: str, stored: str) -> bool: + try: + return password_hasher.verify(stored, password) + except (VerificationError, InvalidHashError): + return False + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() diff --git a/src/core/session_store.py b/src/core/session_store.py new file mode 100644 index 0000000..1d7083f --- /dev/null +++ b/src/core/session_store.py @@ -0,0 +1,55 @@ +"""Сессии в Redis: абсолютный срок жизни, без продления при чтении.""" + +import json +from dataclasses import asdict +from datetime import datetime, timezone +from uuid import UUID + +from redis.exceptions import ResponseError + +from src.core import cache +from src.service.dto import StoredSessionDTO + + +def session_key(token_hash: str) -> str: + return f"session:v1:{token_hash}" + + +async def create_session(token_hash: str, session: StoredSessionDTO) -> None: + await cache.get_client().set( + session_key(token_hash), + json.dumps(asdict(session), default=str), + pxat=int(session.expires_at.timestamp() * 1000), + ) + + +async def fetch_session(token_hash: str) -> StoredSessionDTO | None: + key = session_key(token_hash) + try: + raw = await cache.get_client().get(key) + if raw is None: + return None + data = json.loads(raw) + if ( + not isinstance(data, dict) + or set(data) != {"user_id", "expires_at"} + or not all(isinstance(value, str) for value in data.values()) + ): + raise ValueError + session = StoredSessionDTO( + UUID(data["user_id"]), datetime.fromisoformat(data["expires_at"]) + ) + if session.expires_at.tzinfo is None or session.expires_at <= datetime.now(timezone.utc): + raise ValueError + return session + except ResponseError as error: + if not str(error).startswith("WRONGTYPE"): + raise + except (ValueError, KeyError, TypeError, UnicodeError): + pass + await cache.get_client().delete(key) + return None + + +async def delete_session(token_hash: str) -> None: + await cache.get_client().delete(session_key(token_hash)) diff --git a/src/core/settings.py b/src/core/settings.py new file mode 100644 index 0000000..cbe3b65 --- /dev/null +++ b/src/core/settings.py @@ -0,0 +1,130 @@ +from typing import Literal, Self + +from pydantic import Field, SecretStr, field_validator, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class AppSettings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + hide_input_in_errors=True, + ) + + otel_enabled: bool = True + otel_exporter_otlp_endpoint: str = "http://otel-collector:4318" + otel_service_name: str = "messenger-api" + + app_host: str = "0.0.0.0" + app_port: int = Field(default=8080, ge=1, le=65535) + app_env: Literal["lab", "production"] = "lab" + app_workers: int = Field(default=1, ge=1) + shutdown_timeout_seconds: int = Field(default=10, gt=0) + healthcheck_timeout_seconds: float = Field(default=2, gt=0) + http_max_body_bytes: int = Field(default=65536, ge=65536) + http_request_timeout_seconds: float = Field(default=5, gt=0) + http_body_timeout_seconds: float = Field(default=5, gt=0) + http_concurrency_limit: int = Field(default=64, gt=0) + authorization_max_length: int = Field(default=2048, ge=1024) + cursor_max_length: int = Field(default=1024, ge=512) + password_hash_concurrency: int = Field(default=2, gt=0) + + s3_endpoint: str = "http://minio:9000" + s3_region: str = "us-east-1" + s3_bucket: str = "messenger-attachments" + s3_access_key_id: SecretStr = SecretStr("messenger-api") + s3_secret_access_key: SecretStr = SecretStr("local-messenger-api-password") + minio_root_user: SecretStr = SecretStr("messenger") + minio_root_password: SecretStr = SecretStr("local-messenger-s3-password") + s3_reconcile_access_key_id: SecretStr = SecretStr("messenger-reconciler") + s3_reconcile_secret_access_key: SecretStr = SecretStr("local-messenger-reconciler-password") + reconcile_grace_seconds: int = Field(default=300, ge=1) + s3_timeout_seconds: float = Field(default=5, gt=0) + max_upload_bytes: int = Field(default=10485760, gt=0, le=10485760) + max_image_pixels: int = Field(default=40000000, gt=0, le=40000000) + upload_concurrency: int = Field(default=1, gt=0) + upload_timeout_seconds: float = Field(default=30, gt=0) + + s3_worker_access_key_id: SecretStr = SecretStr("messenger-worker") + s3_worker_secret_access_key: SecretStr = SecretStr("local-messenger-worker-password") + + broker_url: SecretStr = SecretStr("redis://broker:6379/0") + worker_concurrency: int = Field(default=1, ge=1, le=8) + job_max_attempts: int = Field(default=5, ge=1, le=20) + job_timeout_seconds: int = Field(default=30, ge=1, le=120) + job_lease_seconds: int = Field(default=120, ge=5) + job_retry_base_seconds: float = Field(default=2, gt=0) + job_retry_cap_seconds: float = Field(default=60, gt=0) + staging_retention_seconds: int = Field(default=604800, ge=1) + worker_memory_mb: int = Field(default=768, ge=256) + pipeline_fault_injection: bool = False + + database_url: SecretStr + database_pool_size: int = Field(default=10, gt=0) + database_max_overflow: int = Field(default=0, ge=0) + database_pool_timeout_seconds: float = Field(default=5, gt=0) + database_connect_timeout_seconds: float = Field(default=5, gt=0) + database_command_timeout_seconds: float = Field(default=5, gt=0) + database_statement_timeout_ms: int = Field(default=5000, gt=0) + database_pool_pre_ping: bool = True + + redis_url: SecretStr + redis_max_connections: int = Field(default=10, gt=0) + redis_connect_timeout_seconds: float = Field(default=0.3, gt=0) + redis_read_timeout_seconds: float = Field(default=0.3, gt=0) + redis_retry_on_timeout: bool = False + profile_cache_ttl_seconds: int = Field(default=60, gt=0) + + admin_username: str = Field(default="admin", pattern=r"^[a-z0-9_]{3,32}$") + admin_password: SecretStr = Field( + default=SecretStr("admin-password-123"), + min_length=12, + max_length=128, + ) + admin_display_name: str = Field(default="Администратор", min_length=1, max_length=100) + + session_ttl_seconds: int = Field(default=3600, gt=0) + session_token_bytes: int = Field(default=32, ge=32) + argon2_time_cost: int = Field(default=3, gt=0) + argon2_memory_cost: int = Field(default=65536, ge=8192) + argon2_parallelism: int = Field(default=4, gt=0) + argon2_salt_len: int = Field(default=16, ge=16) + argon2_hash_len: int = Field(default=32, ge=32) + + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" + log_format: str = "%(asctime)s %(levelname)s %(name)s: %(message)s" + + @field_validator("admin_password") + @classmethod + def admin_password_must_not_be_blank(cls, value: SecretStr) -> SecretStr: + if not value.get_secret_value().strip(): + raise ValueError("Пароль администратора не может быть пробельным") + value.get_secret_value().encode("utf-8") + return value + + @field_validator("admin_display_name") + @classmethod + def admin_display_name_must_be_valid(cls, value: str) -> str: + if not value.strip() or "\x00" in value: + raise ValueError("Некорректное имя администратора") + return value + + @model_validator(mode="after") + def production_must_use_custom_password(self) -> Self: + if self.app_env == "production" and self.admin_password.get_secret_value() == ( + "admin-password-123" + ): + raise ValueError("Для production требуется собственный ADMIN_PASSWORD") + return self + + @property + def sqlalchemy_database_url(self) -> str: + return self.database_url.get_secret_value().replace( + "postgresql://", + "postgresql+asyncpg://", + 1, + ) + + +settings = AppSettings() diff --git a/src/core/storage.py b/src/core/storage.py new file mode 100644 index 0000000..8e6c900 --- /dev/null +++ b/src/core/storage.py @@ -0,0 +1,96 @@ +"""S3 clients with separate roles, bounded socket timeouts and request metrics.""" + +from collections import Counter +from dataclasses import dataclass, field +from functools import lru_cache +from urllib.parse import parse_qs, urlsplit + +import urllib3 +from minio import Minio +from minio.error import InvalidResponseError, S3Error, ServerError +from opentelemetry.trace import SpanKind, Status, StatusCode + +from src.core.errors import ApiError +from src.core.settings import settings +from src.observability import signals + + +@dataclass +class StorageMetrics: + requests: Counter = field(default_factory=Counter) + + +class StoragePool(urllib3.PoolManager): + def __init__(self, *args, metrics=None, **kwargs): + super().__init__(*args, **kwargs) + self.metrics = metrics + + def urlopen(self, method, url, *args, **kwargs): + if self.metrics is not None: + query = parse_qs(urlsplit(url).query, keep_blank_values=True) + operation = ( + "LIST" + if "list-type" in query + else ("VERSIONING" if "versioning" in query else method) + ) + self.metrics.requests[operation] += 1 + with signals.tracer.start_as_current_span( + "s3." + method, + kind=SpanKind.CLIENT, + attributes={"storage.operation": method}, + record_exception=False, + set_status_on_exception=False, + ) as span: + try: + response = super().urlopen(method, url, *args, **kwargs) + span.set_attribute("http.response.status_code", response.status) + if response.status >= 500: + span.set_status(Status(StatusCode.ERROR)) + return response + except Exception as error: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute("error.type", type(error).__name__) + raise + + +def make_storage(*, role="api", metrics=None) -> Minio: + endpoint = urlsplit(settings.s3_endpoint) + if endpoint.scheme not in {"http", "https"} or not endpoint.netloc: + raise ValueError("S3_ENDPOINT должен быть HTTP(S) URL") + credentials = { + "api": (settings.s3_access_key_id, settings.s3_secret_access_key), + "reconciler": ( + settings.s3_reconcile_access_key_id, + settings.s3_reconcile_secret_access_key, + ), + "worker": (settings.s3_worker_access_key_id, settings.s3_worker_secret_access_key), + "bootstrap": (settings.minio_root_user, settings.minio_root_password), + } + access_key, secret_key = credentials[role] + return Minio( + endpoint.netloc, + access_key=access_key.get_secret_value(), + secret_key=secret_key.get_secret_value(), + secure=endpoint.scheme == "https", + region=settings.s3_region, + http_client=StoragePool( + metrics=metrics, + timeout=urllib3.Timeout( + connect=settings.s3_timeout_seconds, read=settings.s3_timeout_seconds + ), + retries=False, + maxsize=8, + ), + ) + + +@lru_cache(maxsize=1) +def get_storage() -> Minio: + return make_storage() + + +def unavailable(error: Exception) -> ApiError: + return ApiError(503, "unavailable", "Хранилище файлов временно недоступно") + + +STORAGE_ERRORS = (S3Error, InvalidResponseError, ServerError, urllib3.exceptions.HTTPError, OSError) diff --git a/src/core/storage_bootstrap.py b/src/core/storage_bootstrap.py new file mode 100644 index 0000000..f2fe5ef --- /dev/null +++ b/src/core/storage_bootstrap.py @@ -0,0 +1,96 @@ +"""Подготовка приватного bucket и отдельных пользователей MinIO.""" + +import json +from pathlib import Path +from string import Template +from urllib.parse import urlsplit + +import urllib3 +from minio import Minio +from minio.credentials import StaticProvider +from minio.minioadmin import MinioAdmin +from pydantic import SecretStr + +from src.core.settings import settings +from src.core.storage import make_storage + +POLICY_DIRECTORY = Path(__file__).resolve().parents[2] / "infra" / "s3" + + +def create_admin_client() -> MinioAdmin: + endpoint = urlsplit(settings.s3_endpoint) + return MinioAdmin( + endpoint=endpoint.netloc, + secure=endpoint.scheme == "https", + region=settings.s3_region, + credentials=StaticProvider( + settings.minio_root_user.get_secret_value(), + settings.minio_root_password.get_secret_value(), + ), + http_client=urllib3.PoolManager( + timeout=urllib3.Timeout( + connect=settings.s3_timeout_seconds, read=settings.s3_timeout_seconds + ), + retries=False, + ), + ) + + +def ensure_private_bucket(storage: Minio) -> None: + if not storage.bucket_exists(settings.s3_bucket): + storage.make_bucket(settings.s3_bucket, location=settings.s3_region) + storage.delete_bucket_policy(settings.s3_bucket) + + +def load_policy(filename: str) -> dict: + template = Template((POLICY_DIRECTORY / filename).read_text(encoding="utf-8")) + return json.loads(template.substitute(bucket=settings.s3_bucket)) + + +def configure_user( + admin: MinioAdmin, + *, + access_key: SecretStr, + secret_key: SecretStr, + policy_name: str, + policy: dict, +) -> None: + username = access_key.get_secret_value() + if username == settings.minio_root_user.get_secret_value(): + raise ValueError("Пользователь приложения не должен совпадать с владельцем MinIO") + admin.user_add(username, secret_key.get_secret_value()) + admin.policy_add(policy_name, policy=policy) + admin.policy_set(policy_name, user=username) + + +def main() -> None: + storage = make_storage(role="bootstrap") + admin = create_admin_client() + ensure_private_bucket(storage) + + configure_user( + admin, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + policy_name="messenger-api", + policy=load_policy("api-policy.json"), + ) + configure_user( + admin, + access_key=settings.s3_reconcile_access_key_id, + secret_key=settings.s3_reconcile_secret_access_key, + policy_name="messenger-reconciler", + policy=load_policy("reconciler-policy.json"), + ) + configure_user( + admin, + access_key=settings.s3_worker_access_key_id, + secret_key=settings.s3_worker_secret_access_key, + policy_name="messenger-worker", + policy=load_policy("worker-policy.json"), + ) + print("MinIO настроен: приватный bucket, пользователи API и reconciler") + + +if __name__ == "__main__": + main() diff --git a/src/core/upload_coordination.py b/src/core/upload_coordination.py new file mode 100644 index 0000000..b177d67 --- /dev/null +++ b/src/core/upload_coordination.py @@ -0,0 +1,21 @@ +"""One transaction-scoped PostgreSQL lock for all actions on an S3 key. + +All writers must register an intent and use this protocol. The lock is released +by PostgreSQL on commit, rollback, lost connection or process death. +""" + +import hashlib + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + + +def lock_id(key: str) -> int: + return int.from_bytes(hashlib.sha256(key.encode()).digest()[:8], "big", signed=True) + + +async def lock_object(session: AsyncSession, key: str, *, wait: bool = False) -> bool: + if wait: + await session.execute(select(func.pg_advisory_xact_lock(lock_id(key)))) + return True + return bool(await session.scalar(select(func.pg_try_advisory_xact_lock(lock_id(key))))) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..fe83ccb --- /dev/null +++ b/src/main.py @@ -0,0 +1,156 @@ +import logging +import time +import uuid + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from opentelemetry.trace import SpanKind, Status, StatusCode +from redis.exceptions import RedisError +from sqlalchemy.exc import DBAPIError +from sqlalchemy.exc import TimeoutError as DatabaseTimeoutError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from src.api.router import router as api_router +from src.core import readiness +from src.core.callback import lifespan +from src.core.errors import ApiError, error_response +from src.core.request_deadline import RequestDeadlineMiddleware +from src.core.request_limits import RequestLimitsMiddleware +from src.core.settings import settings +from src.observability import signals + +logging.basicConfig(level=settings.log_level, format=settings.log_format) + +signals.setup("messenger-api") + +INSTANCE_ID = str(uuid.uuid4()) + +app = FastAPI(lifespan=lifespan) +app.add_middleware(RequestLimitsMiddleware) +app.add_middleware(RequestDeadlineMiddleware) +app.include_router(api_router) +logger = logging.getLogger(__name__) + + +@app.middleware("http") +async def add_request_id(request: Request, call_next): + with signals.tracer.start_as_current_span( + "HTTP", + context=signals.parent( + { + "traceparent": request.headers.get("traceparent"), + "tracestate": request.headers.get("tracestate"), + } + ), + kind=SpanKind.SERVER, + record_exception=False, + set_status_on_exception=False, + ) as span: + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started_at = time.perf_counter() + + try: + response = await call_next(request) + except Exception: + logger.exception( + "Необработанная ошибка запроса", + extra={"request_id": request_id}, + ) + + response = JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "Внутренняя ошибка сервера", + "request_id": request_id, + } + }, + ) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + if request.url.path.startswith("/api/v1/"): + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + route = getattr(request.scope.get("route"), "path", "unmatched") + if request.url.path.startswith("/api/v1/") and not route.startswith("/api/v1/"): + route = "/api/v1/" + route.lstrip("/") + attributes = { + "method": request.method + if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "OPTIONS", "HEAD"} + else "OTHER", + "route": route, + "status_class": str(response.status_code // 100) + "xx", + } + elapsed = time.perf_counter() - started_at + signals.http_requests.add(1, attributes) + signals.http_duration.record(elapsed, attributes) + span.update_name(attributes["method"] + " " + route) + span.set_attributes( + { + "http.request.method": attributes["method"], + "http.route": route, + "http.response.status_code": response.status_code, + } + ) + if response.status_code >= 500: + span.set_status(Status(StatusCode.ERROR)) + response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x") + signals.event( + "http.completed", + request_id=request_id, + method=attributes["method"], + route=route, + status=response.status_code, + duration_seconds=elapsed, + ) + return response + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + response = error_response(request, exc.status_code, exc.code, exc.message) + if exc.status_code == 401: + response.headers["WWW-Authenticate"] = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == "/api/v1/auth/sessions" + else 'Bearer realm="messenger"' + ) + return response + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Некорректные параметры запроса") + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse: + code = "not_found" if exc.status_code == 404 else "invalid_request" + message = "Объект не найден" if exc.status_code == 404 else "Некорректный запрос" + return error_response(request, exc.status_code, code, message) + + +@app.get("/health/live") +async def get_health(): + return { + "status": "ok", + } + + +@app.get("/health/ready") +async def get_ready(): + await readiness.check() + return {"status": "ok"} + + +async def handle_database_error(request: Request, exc: Exception) -> JSONResponse: + logger.warning("Хранилище временно недоступно: тип_ошибки=%s", type(exc).__name__) + return error_response(request, 503, "unavailable", "Хранилище временно недоступно") + + +for error_type in (RedisError, DBAPIError, DatabaseTimeoutError, OSError, TimeoutError): + app.add_exception_handler(error_type, handle_database_error) diff --git a/src/maintenance/__init__.py b/src/maintenance/__init__.py new file mode 100644 index 0000000..42c06c7 --- /dev/null +++ b/src/maintenance/__init__.py @@ -0,0 +1 @@ +"""Operator maintenance commands; no public HTTP endpoints.""" diff --git a/src/maintenance/reconcile.py b/src/maintenance/reconcile.py new file mode 100644 index 0000000..f537182 --- /dev/null +++ b/src/maintenance/reconcile.py @@ -0,0 +1,361 @@ +"""Bounded, repeatable S3/PostgreSQL reconciliation. Default mode is dry-run.""" + +import argparse +import asyncio +import hashlib +import json +import resource +import time +from collections import Counter +from datetime import timedelta +from uuid import UUID + +from anyio import to_thread +from minio.error import S3Error +from sqlalchemy import event, func, or_, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, StorageFinding, UploadIntent +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import StorageMetrics, make_storage +from src.core.upload_coordination import lock_object + + +class Reconciler: + def __init__( + self, + factory, + storage, + *, + apply=False, + grace_seconds=300, + prefix="originals/", + sample_limit=50, + ): + if not prefix.startswith("originals/") or not prefix.endswith("/"): + raise ValueError("Scope must be originals/ or a narrower prefix ending with /") + if grace_seconds < 1: + raise ValueError("Grace period must be positive") + self.factory = factory + self.storage = storage + self.apply = apply + self.grace = timedelta(seconds=grace_seconds) + self.prefix = prefix + self.sample_limit = sample_limit + self.counts = Counter() + self.samples = [] + self.content_bytes_read = 0 + + def note(self, action, key): + self.counts[action] += 1 + if len(self.samples) < self.sample_limit: + self.samples.append({"action": action, "object_key": key}) + + async def stat(self, key): + try: + return await to_thread.run_sync( + lambda: self.storage.stat_object(settings.s3_bucket, key) + ) + except S3Error as error: + # Bucket absence, AccessDenied, timeout and server failure must abort. + if error.code not in {"NoSuchKey", "NoSuchObject"}: + raise + return None + + async def observe(self, session, key, kind, now, fingerprint=None): + finding = await session.get(StorageFinding, (key, kind)) + if ( + finding is None + or finding.resolved_at is not None + or (kind == "orphan" and finding.fingerprint != fingerprint) + ): + if not self.apply: + return None + if finding is None: + finding = StorageFinding(object_key=key, kind=kind) + session.add(finding) + finding.first_seen = now + finding.resolved_at = None + finding.fingerprint = fingerprint + if self.apply: + finding.last_seen = now + return finding + + async def resolve(self, session, key, kind, now): + if self.apply: + finding = await session.get(StorageFinding, (key, kind)) + if finding is not None and finding.resolved_at is None: + finding.resolved_at = now + finding.last_seen = now + + async def orphan(self, key): + async with self.factory.begin() as session: + if not await lock_object(session, key): + self.note("protected_active", key) + return + now = await session.scalar(select(func.clock_timestamp())) + attachment = await session.scalar( + select(Attachment.id).where( + or_(Attachment.object_key == key, Attachment.staging_key == key) + ) + ) + if attachment is not None: + self.note("protected_referenced", key) + await self.resolve(session, key, "orphan", now) + return + obj = await self.stat(key) + if obj is None: + await self.resolve(session, key, "orphan", now) + return + # Opaque ETag identifies a changed object; it is NOT treated as SHA-256. + fingerprint = f"{obj.etag}:{obj.size}:{obj.last_modified.isoformat()}" + finding = await self.observe(session, key, "orphan", now, fingerprint) + intent = await session.get(UploadIntent, key) + cutoff = now - self.grace + if obj.last_modified > cutoff or (intent is not None and intent.created_at > cutoff): + self.note("protected_fresh", key) + return + if not self.apply: + self.note("would_delete_orphan", key) + return + # Registration, PUT, publication and cleanup all take this same lock. + # Tombstone also prevents a delayed pending uploader from publishing. + if intent is not None: + intent.state = "abandoned" + await session.flush() + await to_thread.run_sync(lambda: self.storage.remove_object(settings.s3_bucket, key)) + finding.resolved_at = now + self.note("deleted_orphan", key) + + async def content_matches(self, attachment): + def verify(): + response = self.storage.get_object(settings.s3_bucket, attachment.object_key) + digest = hashlib.sha256() + size = 0 + try: + while chunk := response.read(65536): + size += len(chunk) + if size > attachment.size_bytes: + return False, size + digest.update(chunk) + finally: + response.close() + response.release_conn() + return size == attachment.size_bytes and digest.hexdigest() == attachment.sha256, size + + valid, size = await to_thread.run_sync(verify) + self.content_bytes_read += size + return valid + + async def check_attachment(self, attachment_id, key): + async with self.factory.begin() as session: + if not await lock_object(session, key): + self.note("protected_active", key) + return + attachment = await session.get(Attachment, attachment_id) + if attachment is None or attachment.object_key != key or attachment.status != "ready": + return + now = await session.scalar(select(func.clock_timestamp())) + obj = await self.stat(key) + if obj is None: + await self.observe(session, key, "missing", now) + if attachment.storage_missing_since is None: + if self.apply: + attachment.storage_missing_since = now + self.note("marked_missing" if self.apply else "would_mark_missing", key) + else: + self.note("missing_unchanged", key) + return + if attachment.storage_missing_since is not None: + if obj.size != attachment.size_bytes or not await self.content_matches(attachment): + self.note("restore_checksum_mismatch", key) + return + if self.apply: + attachment.storage_missing_since = None + self.note("recovered" if self.apply else "would_recover", key) + else: + self.note("healthy", key) + await self.resolve(session, key, "missing", now) + + async def stale_intent(self, key): + async with self.factory.begin() as session: + if not await lock_object(session, key): + self.note("protected_active_intent", key) + return + intent = await session.get(UploadIntent, key) + if intent is None or intent.state != "pending": + return + now = await session.scalar(select(func.clock_timestamp())) + if intent.created_at > now - self.grace: + return + attached = await session.scalar( + select(Attachment.id).where( + or_(Attachment.object_key == key, Attachment.staging_key == key) + ) + ) + if attached is not None: + if self.apply: + intent.state = "committed" + self.note("intent_has_attachment", key) + elif await self.stat(key) is None: + await self.resolve(session, key, "orphan", now) + if self.apply: + intent.state = "abandoned" + self.note("abandoned_empty_intent" if self.apply else "would_abandon_intent", key) + + async def run(self): + # SDK lazily fetches ListObjectsV2 pages; never collect the whole bucket. + objects = self.storage.list_objects(settings.s3_bucket, prefix=self.prefix, recursive=True) + sentinel = object() + while True: + obj = await to_thread.run_sync(lambda: next(objects, sentinel)) + if obj is sentinel: + break + self.counts["objects_scanned"] += 1 + await self.orphan(obj.object_name) + + after: UUID | None = None + while True: + async with self.factory() as session: + query = ( + select(Attachment.id, Attachment.object_key) + .where(Attachment.object_key.startswith(self.prefix, autoescape=True)) + .order_by(Attachment.id) + .limit(200) + ) + if after is not None: + query = query.where(Attachment.id > after) + batch = (await session.execute(query)).all() + if not batch: + break + for aid, key in batch: + self.counts["attachments_scanned"] += 1 + await self.check_attachment(aid, key) + after = batch[-1].id + + # Pending attempts without any visible object also need bounded cleanup. + after_key = "" + while True: + async with self.factory() as session: + batch = list( + await session.scalars( + select(UploadIntent.object_key) + .where( + UploadIntent.state == "pending", + UploadIntent.object_key > after_key, + UploadIntent.object_key.startswith(self.prefix, autoescape=True), + ) + .order_by(UploadIntent.object_key) + .limit(200) + ) + ) + if not batch: + break + for key in batch: + self.counts["pending_intents_scanned"] += 1 + await self.stale_intent(key) + after_key = batch[-1] + + after_key = "" + while True: + async with self.factory() as session: + batch = list( + await session.scalars( + select(StorageFinding.object_key) + .where( + StorageFinding.kind == "orphan", + StorageFinding.resolved_at.is_(None), + StorageFinding.first_seen <= func.clock_timestamp() - self.grace, + StorageFinding.object_key > after_key, + StorageFinding.object_key.startswith(self.prefix, autoescape=True), + ) + .order_by(StorageFinding.object_key) + .limit(200) + ) + ) + if not batch: + break + for key in batch: + await self.orphan(key) + after_key = batch[-1] + + return { + "mode": "apply" if self.apply else "dry-run", + "bucket": settings.s3_bucket, + "prefix": self.prefix, + "grace_seconds": self.grace.total_seconds(), + "counts": dict(self.counts), + "samples": self.samples, + "content_bytes_read": self.content_bytes_read, + } + + +async def execute(args): + engine = create_engine() + db_statements = 0 + + @event.listens_for(engine.sync_engine, "before_cursor_execute") + def count_statements(*_): + nonlocal db_statements + db_statements += 1 + + storage_metrics = StorageMetrics() + storage = make_storage(role="reconciler", metrics=storage_metrics) + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + while True: + started = time.perf_counter() + db_statements = 0 + storage_metrics.requests.clear() + report = {"mode": "apply" if args.apply else "dry-run", "prefix": args.prefix} + try: + # This implementation intentionally supports an unversioned bucket. + versioning = await to_thread.run_sync( + lambda: storage.get_bucket_versioning(settings.s3_bucket) + ) + if versioning.status in {"Enabled", "Suspended"}: + raise ValueError("Versioned buckets require version-aware reconciliation") + report = await Reconciler( + factory, + storage, + apply=args.apply, + grace_seconds=args.grace_seconds, + prefix=args.prefix, + ).run() + report["result"] = "ok" + except Exception as error: + # Do not print connection strings, SDK bodies or credentials. + report.update(result="error", error_type=type(error).__name__) + report.update( + elapsed_seconds=round(time.perf_counter() - started, 6), + db_statements=db_statements, + s3_requests=dict(storage_metrics.requests), + process_peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + ) + print(json.dumps(report, ensure_ascii=False), flush=True) + if not args.interval: + return 0 if report["result"] == "ok" else 1 + await asyncio.sleep(args.interval) + finally: + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", action="store_true", help="Apply safe repairs; otherwise dry-run" + ) + parser.add_argument("--prefix", default="originals/") + parser.add_argument("--grace-seconds", type=int, default=settings.reconcile_grace_seconds) + parser.add_argument( + "--interval", type=int, default=0, help="Repeat every N seconds; 0 runs once" + ) + args = parser.parse_args() + if args.interval < 0: + parser.error("interval must be nonnegative") + raise SystemExit(asyncio.run(execute(args))) + + +if __name__ == "__main__": + main() diff --git a/src/observability/__init__.py b/src/observability/__init__.py new file mode 100644 index 0000000..7015ac2 --- /dev/null +++ b/src/observability/__init__.py @@ -0,0 +1 @@ +"""Bounded OTLP signals; telemetry is never a business dependency.""" diff --git a/src/observability/signals.py b/src/observability/signals.py new file mode 100644 index 0000000..9ecd3b9 --- /dev/null +++ b/src/observability/signals.py @@ -0,0 +1,182 @@ +"""Low-cardinality metrics, sanitized events and explicit W3C propagation.""" + +import atexit +import json +import logging +import os +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import SpanLimits, TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from src.core.settings import settings + +tracer = trace.get_tracer("messenger", "1.6.0") +meter = metrics.get_meter("messenger", "1.6.0") +http_requests = meter.create_counter("http.server.requests") +http_duration = meter.create_histogram("http.server.duration", unit="s") +jobs = meter.create_counter("worker.jobs") +job_duration = meter.create_histogram("worker.job.duration", unit="s") +ready_duration = meter.create_histogram("attachment.ready.duration", unit="s") +worker_busy = meter.create_up_down_counter("worker.busy") +queue_depth = meter.create_gauge("queue.depth") +queue_age = meter.create_gauge("queue.oldest.age", unit="s") +db_available = meter.create_gauge("db.available") +propagator = TraceContextTextMapPropagator() +configured = False +logger = logging.getLogger("messenger.events") +logger.setLevel(logging.INFO) +logger.propagate = False + + +def carrier(): + result = {} + propagator.inject(result) + return result + + +def parent(carrier): + return propagator.extract( + {k: v for k, v in carrier.items() if v and k in {"traceparent", "tracestate"}} + ) + + +def event(name, **fields): + # Callers pass explicit safe fields. Never forward exception text, HTTP headers, + # SQL statements/parameters, URL, filename or arbitrary application objects. + context = trace.get_current_span().get_span_context() + body = dict( + event=name, + timestamp=datetime.now(timezone.utc).isoformat(), + severity="INFO", + service_name=settings.otel_service_name, + trace_id=format(context.trace_id, "032x"), + span_id=format(context.span_id, "016x"), + **fields, + ) + logger.info(json.dumps(body, ensure_ascii=False), extra={"event": name}) + + +def setup(service=None): + global configured + if configured: + return + configured = True + if service: + settings.otel_service_name = service + stdout = logging.StreamHandler() + stdout.setFormatter(logging.Formatter("%(message)s")) + logger.addHandler(stdout) + if not settings.otel_enabled: + return + resource = Resource.create( + { + "service.name": settings.otel_service_name, + "service.version": "1.6.0", + "service.instance.id": str(uuid4()), + "deployment.environment.name": "lab", + } + ) + endpoint = settings.otel_exporter_otlp_endpoint.rstrip("/") + spans = TracerProvider( + resource=resource, span_limits=SpanLimits(max_attributes=32, max_events=8) + ) + spans.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=1), + max_queue_size=256, + max_export_batch_size=32, + schedule_delay_millis=500, + export_timeout_millis=1500, + ) + ) + trace.set_tracer_provider(spans) + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=1), + max_queue_size=256, + max_export_batch_size=32, + schedule_delay_millis=500, + export_timeout_millis=1500, + ) + ) + set_logger_provider(logs) + logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=1), + export_interval_millis=2000, + export_timeout_millis=1500, + ) + provider = MeterProvider( + resource=resource, + metric_readers=[reader], + views=[ + View( + instrument_type=metrics.Histogram, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=( + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 2.5, + 5, + 10, + 30, + 60, + 120, + ) + ), + ) + ], + ) + metrics.set_meter_provider(provider) + + def rss(_): + resident = int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE") + yield metrics.Observation(resident) + + meter.create_observable_gauge("process.memory.rss", callbacks=[rss], unit="By") + meter.create_observable_counter( + "process.cpu.time", + callbacks=[lambda _: [metrics.Observation(time.process_time())]], + unit="s", + ) + if settings.otel_service_name == "messenger-worker": + meter.create_observable_gauge( + "worker.capacity", + callbacks=[lambda _: [metrics.Observation(settings.worker_concurrency)]], + ) + # Explicit bounds apply to the asynchronous queues; dropping telemetry is + # preferable to delaying business requests. Shutdown runs outside requests. + atexit.register(lambda: spans.shutdown()) + + +def register_pool(engine): + pool = engine.sync_engine.pool + meter.create_observable_gauge( + "db.pool.used", callbacks=[lambda _: [metrics.Observation(pool.checkedout())]] + ) + meter.create_observable_gauge( + "db.pool.waiting", callbacks=[lambda _: [metrics.Observation(getattr(pool, "waiting", 0))]] + ) diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py new file mode 100644 index 0000000..e43076b --- /dev/null +++ b/src/pipeline/__init__.py @@ -0,0 +1 @@ +"""Durable asynchronous attachment processing.""" diff --git a/src/pipeline/admin.py b/src/pipeline/admin.py new file mode 100644 index 0000000..7405713 --- /dev/null +++ b/src/pipeline/admin.py @@ -0,0 +1,113 @@ +"""Local operator CLI: failed registry and audited, explicit replay.""" + +import argparse +import asyncio +import json +from functools import lru_cache +from uuid import UUID, uuid4 + +from anyio import to_thread +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob, JobReplay +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.pipeline.common import now + + +@lru_cache(maxsize=1) +def worker_storage(): + return make_storage(role="worker") + + +async def execute(args): + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + async with factory.begin() as session: + if args.command == "failed": + jobs = await session.scalars( + select(AttachmentJob) + .where(AttachmentJob.state == "failed") + .order_by(AttachmentJob.updated_at) + .limit(100) + ) + print( + json.dumps( + [ + dict( + job_id=str(j.id), + attachment_id=str(j.attachment_id), + attempt=j.attempt, + error_code=j.error_code, + ) + for j in jobs + ] + ) + ) + return + job = await session.scalar( + select(AttachmentJob).where(AttachmentJob.id == args.job_id).with_for_update() + ) + if job is None or job.state != "failed": + raise ValueError("Replay accepts only an existing failed job") + attachment = await session.scalar( + select(Attachment).where(Attachment.id == job.attachment_id).with_for_update() + ) + if not attachment.staging_key: + raise ValueError("Staging retention expired; replay unavailable") + await to_thread.run_sync( + lambda: worker_storage().stat_object(settings.s3_bucket, attachment.staging_key) + ) + when = await now(session) + job.attempt += 1 + job.cycle_attempt = 1 + job.state = attachment.status = "queued" + job.error_code = attachment.error_code = None + job.published_at = job.claim_token = job.lease_until = None + job.next_run_at = job.updated_at = attachment.updated_at = when + session.add( + JobReplay( + id=uuid4(), + job_id=job.id, + operator=args.operator, + reason=args.reason, + created_at=when, + attempt=job.attempt, + ) + ) + print(json.dumps(dict(job_id=str(job.id), status="queued", attempt=job.attempt))) + finally: + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("failed") + replay = sub.add_parser("replay") + replay.add_argument("job_id", type=UUID) + replay.add_argument("--operator", required=True) + replay.add_argument("--reason", required=True) + args = parser.parse_args() + if args.command == "replay" and ( + not 1 <= len(args.operator.strip()) <= 100 or not 1 <= len(args.reason.strip()) <= 500 + ): + parser.error("operator: 1–100; reason: 1–500 nonblank characters") + try: + asyncio.run(execute(args)) + except Exception as error: + print( + json.dumps( + dict( + error=type(error).__name__, message="Replay rejected or dependency unavailable" + ) + ) + ) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/src/pipeline/common.py b/src/pipeline/common.py new file mode 100644 index 0000000..d3f9557 --- /dev/null +++ b/src/pipeline/common.py @@ -0,0 +1,58 @@ +"""Shared state transitions; every acknowledgement follows a DB commit.""" + +import random +from datetime import timedelta + +from redis.asyncio import Redis +from sqlalchemy import func, select + +from src.core.settings import settings +from src.observability import signals + +STREAM = "attachment-jobs-v1" +GROUP = "image-workers-v1" + + +def broker(): + return Redis.from_url( + settings.broker_url.get_secret_value(), + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=3, + ) + + +async def now(session): + return await session.scalar(select(func.clock_timestamp())) + + +def retry(job, attachment, when, code, permanent=False): + context = signals.carrier() + if context.get("traceparent"): + job.traceparent = context["traceparent"] + job.tracestate = context.get("tracestate") + signals.event( + "job.retry" + if not permanent and job.cycle_attempt < settings.job_max_attempts + else "job.failed", + attempt=job.attempt, + error_code=code, + ) + job.claim_token = None + job.lease_until = None + job.updated_at = attachment.updated_at = when + job.error_code = code + if permanent or job.cycle_attempt >= settings.job_max_attempts: + job.state = attachment.status = "failed" + attachment.error_code = code if permanent else "retry_exhausted" + else: + cap = min( + settings.job_retry_cap_seconds, + settings.job_retry_base_seconds * 2 ** (job.cycle_attempt - 1), + ) + job.next_run_at = when + timedelta(seconds=random.uniform(cap / 2, cap)) + job.attempt += 1 + job.cycle_attempt += 1 + job.state = attachment.status = "queued" + job.published_at = None + attachment.error_code = None diff --git a/src/pipeline/dispatcher.py b/src/pipeline/dispatcher.py new file mode 100644 index 0000000..2ea800e --- /dev/null +++ b/src/pipeline/dispatcher.py @@ -0,0 +1,197 @@ +"""Transactional outbox relay, expired-lease recovery and staging retention.""" + +import asyncio +import json +import logging +import os +from datetime import datetime, timedelta, timezone +from functools import lru_cache + +from anyio import to_thread +from opentelemetry.trace import SpanKind +from sqlalchemy import func, or_, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob, UploadIntent +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.core.upload_coordination import lock_object +from src.observability import signals +from src.pipeline.common import STREAM, broker, now, retry + + +@lru_cache(maxsize=1) +def worker_storage(): + return make_storage(role="worker") + + +logger = logging.getLogger(__name__) + + +async def dispatch(factory, client): + async with factory.begin() as session: + when = await now(session) + jobs = list( + await session.scalars( + select(AttachmentJob) + .where(AttachmentJob.state == "processing", AttachmentJob.lease_until < when) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for job in jobs: + attachment = await session.get(Attachment, job.attachment_id) + retry(job, attachment, when, "worker_lost") + # Bounded batch; row locks serialize multiple relays. XADD may succeed while + # COMMIT fails: that creates a duplicate, never a missing durable intention. + async with factory.begin() as session: + when = await now(session) + jobs = list( + await session.scalars( + select(AttachmentJob) + .where( + AttachmentJob.state == "queued", + AttachmentJob.published_at.is_(None), + AttachmentJob.next_run_at <= when, + ) + .order_by(AttachmentJob.next_run_at) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for job in jobs: + payload = dict( + schema_version=1, + job_id=str(job.id), + type="attachment.process.v1", + attachment_id=str(job.attachment_id), + pipeline_version=1, + attempt=job.attempt, + created_at=job.created_at.isoformat().replace("+00:00", "Z"), + ) + if settings.pipeline_fault_injection and job.fault == "before_publish": + os._exit(81) + with signals.tracer.start_as_current_span( + "queue.publish", + context=signals.parent( + {"traceparent": job.traceparent, "tracestate": job.tracestate} + ), + kind=SpanKind.PRODUCER, + record_exception=False, + set_status_on_exception=False, + ): + payload.update(signals.carrier()) + await client.xadd(STREAM, {"job": json.dumps(payload)}) + signals.event( + "job.published", attempt=job.attempt, job_type="attachment.process.v1" + ) + if settings.pipeline_fault_injection and job.fault == "after_publish": + os._exit(82) + job.published_at = when + + +async def cleanup(factory): + storage = worker_storage() + async with factory.begin() as session: + when = await now(session) + attachments = list( + await session.scalars( + select(Attachment) + .where( + Attachment.staging_key.is_not(None), + or_( + Attachment.status == "ready", + (Attachment.status == "failed") + & ( + Attachment.updated_at + < when - timedelta(seconds=settings.staging_retention_seconds) + ), + ), + ) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for attachment in attachments: + # Serialize deletion with replay, which locks the attachment too. + if attachment.staging_key != attachment.object_key: + await to_thread.run_sync( + lambda: storage.remove_object(settings.s3_bucket, attachment.staging_key) + ) + attachment.staging_key = None + # Intents also cover a crash after staging PUT, before metadata/job COMMIT. + async with factory.begin() as session: + when = await now(session) + intents = list( + await session.scalars( + select(UploadIntent) + .where( + UploadIntent.object_key.startswith("staging/"), + UploadIntent.state == "pending", + UploadIntent.created_at + < when - timedelta(seconds=settings.reconcile_grace_seconds), + ) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for intent in intents: + if not await lock_object(session, intent.object_key): + continue + attached = await session.scalar( + select(Attachment.id).where(Attachment.staging_key == intent.object_key) + ) + if attached is None: + await to_thread.run_sync( + lambda: storage.remove_object(settings.s3_bucket, intent.object_key) + ) + intent.state = "abandoned" + + +async def main(): + signals.setup("messenger-dispatcher") + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + client = broker() + ticks = 0 + try: + while True: + try: + await dispatch(factory, client) + async with factory() as session: + count, oldest = ( + await session.execute( + select(func.count(), func.min(AttachmentJob.created_at)).where( + AttachmentJob.state.in_(["queued", "processing"]) + ) + ) + ).one() + attrs = {"queue": STREAM} + signals.queue_depth.set(count, attrs) + signals.queue_age.set( + max(0, (datetime.now(timezone.utc) - oldest).total_seconds()) + if oldest + else 0, + attrs, + ) + signals.db_available.set(1) + except Exception as error: + signals.db_available.set(0) + signals.event("relay.error", error_type=type(error).__name__) + logger.warning("relay unavailable: %s", type(error).__name__) + if ticks % 10 == 0: + try: + await cleanup(factory) + except Exception as error: + logger.warning("cleanup unavailable: %s", type(error).__name__) + ticks += 1 + await asyncio.sleep(0.5) + finally: + await client.aclose() + await engine.dispose() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) diff --git a/src/pipeline/images.py b/src/pipeline/images.py new file mode 100644 index 0000000..ee3bb5d --- /dev/null +++ b/src/pipeline/images.py @@ -0,0 +1,95 @@ +"""Isolated image transformation subprocess; originals are never re-encoded.""" + +import hashlib +import json +import os +import resource +import sys +import time +import warnings +from pathlib import Path + +from PIL import Image, ImageOps + + +def dimensions(width, height, limit): + # Contract requires floor, unlike Pillow thumbnail's closest-aspect rounding. + largest = max(width, height) + if largest <= limit: + return width, height + return max(1, width * limit // largest), max(1, height * limit // largest) + + +def transform(source: Path, output: Path, max_pixels: int): + Image.MAX_IMAGE_PIXELS = max_pixels + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(source) as raw: + if raw.format not in {"PNG", "JPEG"}: + raise ValueError("invalid format") + if raw.width * raw.height > max_pixels: + raise ValueError("pixel limit") + raw.verify() + with Image.open(source) as raw: + raw.load() + oriented = ImageOps.exif_transpose(raw) + # All generated variants use an RGB canvas, so no EXIF/XMP/ICC is inherited. + rgba = oriented.convert("RGBA") + image = Image.new("RGB", rgba.size, "white") + image.paste(rgba, mask=rgba.getchannel("A")) + del rgba + variants = [] + for name, size, fmt, quality in ( + ("thumbnail", dimensions(*image.size, 256), "JPEG", 85), + ("crop", (128, 128), "JPEG", 85), + ("optimized", dimensions(*image.size, 1280), "WEBP", 80), + ): + result = ( + ImageOps.fit(image, size, Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + if name == "crop" + else image.resize(size, Image.Resampling.LANCZOS) + ) + options = {"method": 4} if fmt == "WEBP" else {"optimize": True} + target = output / name + result.save(target, fmt, quality=quality, **options) + data = target.read_bytes() + variants.append( + dict( + name=name, + width=result.width, + height=result.height, + content_type="image/webp" if fmt == "WEBP" else "image/jpeg", + size_bytes=len(data), + sha256=hashlib.sha256(data).hexdigest(), + ) + ) + result.close() + image.close() + (output / "manifest.json").write_text(json.dumps(variants)) + + +def main(): + source, output, max_pixels, memory_mb, seconds = sys.argv[1:] + # Linux worker subprocess: hard virtual-memory and CPU ceilings plus parent wall timeout. + memory = int(memory_mb) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (int(seconds), int(seconds) + 1)) + if os.environ.get("PIPELINE_FAULT_INJECTION", "").lower() == "true": + time.sleep(float(os.environ.get("PIPELINE_TEST_IMAGE_DELAY", "0"))) + try: + transform(Path(source), Path(output), int(max_pixels)) + except ( + Image.DecompressionBombError, + Image.DecompressionBombWarning, + ValueError, + OSError, + SyntaxError, + EOFError, + ): + raise SystemExit(2) from None + except MemoryError: + raise SystemExit(3) from None + + +if __name__ == "__main__": + main() diff --git a/src/pipeline/worker.py b/src/pipeline/worker.py new file mode 100644 index 0000000..3188f41 --- /dev/null +++ b/src/pipeline/worker.py @@ -0,0 +1,364 @@ +"""Redis Streams consumer with bounded prefetch, leases and idempotent effects.""" + +import asyncio +import hashlib +import json +import logging +import os +import socket +import sys +import time +from datetime import datetime, timedelta, timezone +from functools import lru_cache +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Literal +from uuid import UUID, uuid4 + +from anyio import to_thread +from opentelemetry.trace import SpanKind, Status, StatusCode +from pydantic import BaseModel, ConfigDict, Field +from redis.exceptions import ResponseError +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.observability import signals +from src.pipeline.common import GROUP, STREAM, broker, now, retry + + +@lru_cache(maxsize=1) +def worker_storage(): + return make_storage(role="worker") + + +logger = logging.getLogger(__name__) + + +class Envelope(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + schema_version: Literal[1] + job_id: str + type: Literal["attachment.process.v1"] + attachment_id: str + pipeline_version: Literal[1] + attempt: int = Field(ge=1) + created_at: str + traceparent: str | None = None + tracestate: str | None = None + + +class PermanentError(Exception): + pass + + +async def transform(source, directory, *, test_delay=False): + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "src.pipeline.images", + str(source), + str(directory), + str(settings.max_image_pixels), + str(settings.worker_memory_mb), + str(settings.job_timeout_seconds), + env=dict( + os.environ, + PIPELINE_TEST_IMAGE_DELAY=str(settings.job_timeout_seconds + 2) if test_delay else "0", + ), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + await asyncio.wait_for(process.wait(), settings.job_timeout_seconds) + finally: + if process.returncode is None: + process.kill() + await process.wait() + if process.returncode == 2: + raise PermanentError("invalid_image") + if process.returncode: + raise PermanentError("image_resource_limit") + return json.loads((directory / "manifest.json").read_text()) + + +def download(attachment, source): + response = worker_storage().get_object(settings.s3_bucket, attachment.staging_key) + digest, size = hashlib.sha256(), 0 + try: + with source.open("wb") as output: + while chunk := response.read(65536): + size += len(chunk) + if size > settings.max_upload_bytes: + raise PermanentError("payload_too_large") + digest.update(chunk) + output.write(chunk) + finally: + response.close() + response.release_conn() + if size != attachment.size_bytes or digest.hexdigest() != attachment.sha256: + raise PermanentError("source_checksum_mismatch") + + +async def verify_ready(attachment): + objects = [(attachment.object_key, attachment.size_bytes)] + [ + (f"variants/{attachment.id}/v1/{item['name']}", item["size_bytes"]) + for item in attachment.variants + ] + if {item["name"] for item in attachment.variants} != {"thumbnail", "crop", "optimized"}: + # Legacy lab4 attachments have no job and never reach this branch. + raise RuntimeError("Incomplete committed pipeline") + for key, size in objects: + stat = await to_thread.run_sync( + lambda: worker_storage().stat_object(settings.s3_bucket, key) + ) + if stat.size != size: + raise RuntimeError("Committed result is missing") + + +async def _process_job(factory, message): + token = uuid4() + async with factory.begin() as session: + job = await session.scalar( + select(AttachmentJob) + .where(AttachmentJob.id == UUID(message.job_id)) + .with_for_update(skip_locked=True) + ) + if job is None: + # Missing jobs are not valid work; busy jobs must remain pending. + exists = await session.get(AttachmentJob, UUID(message.job_id)) + return exists is None + if ( + str(job.attachment_id) != message.attachment_id + or job.pipeline_version != message.pipeline_version + ): + raise PermanentError("envelope_mismatch") + attachment = await session.get(Attachment, job.attachment_id) + if job.state == "ready": + await verify_ready(attachment) + return "noop" + if job.state == "failed" or message.attempt < job.attempt: + return "noop" + if message.attempt != job.attempt: + raise PermanentError("attempt_mismatch") + when = await now(session) + if job.state == "processing" or when < job.next_run_at: + return False + job.state = attachment.status = "processing" + job.claim_token = token + job.lease_until = when + timedelta(seconds=settings.job_lease_seconds) + job.updated_at = attachment.updated_at = when + # Keep the job row locked during bounded side effects. Recovery uses SKIP + # LOCKED, so a live attempt and its replacement cannot publish concurrently. + # A process crash releases the lock; its durable lease is recovered by relay. + async with factory.begin() as session: + job = await session.scalar( + select(AttachmentJob) + .where(AttachmentJob.id == UUID(message.job_id)) + .with_for_update(skip_locked=True) + ) + if job is None or job.claim_token != token or job.state != "processing": + return False + attachment = await session.get(Attachment, job.attachment_id) + try: + if settings.pipeline_fault_injection and job.fault == "temporary": + raise OSError("injected temporary failure") + if settings.pipeline_fault_injection and job.fault == "permanent": + raise PermanentError("invalid_image") + with TemporaryDirectory(prefix="image-job-") as temporary: + directory = Path(temporary) + source = directory / "original" + await to_thread.run_sync(download, attachment, source) + with signals.tracer.start_as_current_span( + "image.transform", record_exception=False + ): + variants = await transform( + source, + directory, + test_delay=settings.pipeline_fault_injection and job.fault == "timeout", + ) + for name, mime in [("original", attachment.content_type)] + [ + (v["name"], v["content_type"]) for v in variants + ]: + path = directory / name + key = ( + attachment.object_key + if name == "original" + else f"variants/{attachment.id}/v1/{name}" + ) + + def put(): + with path.open("rb") as data: + worker_storage().put_object( + settings.s3_bucket, + key, + data, + path.stat().st_size, + content_type=mime, + num_parallel_uploads=1, + ) + + await to_thread.run_sync(put) + if settings.pipeline_fault_injection and job.fault == "after_objects": + os._exit(83) + attachment.storage_missing_since = None + attachment.variants = variants + attachment.status = job.state = "ready" + attachment.error_code = job.error_code = None + job.claim_token = job.lease_until = None + job.updated_at = attachment.updated_at = await now(session) + except Exception as error: + code = ( + str(error) + if isinstance(error, PermanentError) + else ( + "processing_timeout" + if isinstance(error, TimeoutError) + else "processing_unavailable" + ) + ) + retry(job, attachment, await now(session), code, isinstance(error, PermanentError)) + crash_after_commit = ( + settings.pipeline_fault_injection + and job.fault == "after_commit" + and job.state == "ready" + ) + if crash_after_commit: + os._exit(84) + return job.state + + +async def process_job(factory, message): + started = time.perf_counter() + with signals.tracer.start_as_current_span( + "queue.consume", + context=signals.parent( + {"traceparent": message.traceparent, "tracestate": message.tracestate} + ), + kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "redis", + "job.type": message.type, + "job.attempt": message.attempt, + }, + record_exception=False, + set_status_on_exception=False, + ) as span: + signals.worker_busy.add(1) + signals.event("job.started", attempt=message.attempt, job_type=message.type) + try: + result = await _process_job(factory, message) + outcome = {"queued": "retry", "ready": "ready", "failed": "failed"}.get(result) + if outcome: + attributes = {"job_type": message.type, "outcome": outcome} + signals.jobs.add(1, attributes) + signals.job_duration.record(time.perf_counter() - started, attributes) + if outcome != "ready": + span.set_status(Status(StatusCode.ERROR)) + else: + created = datetime.fromisoformat(message.created_at.replace("Z", "+00:00")) + signals.ready_duration.record( + max(0, (datetime.now(timezone.utc) - created).total_seconds()), + {"job_type": message.type}, + ) + signals.event( + "job.completed", + outcome=outcome, + attempt=message.attempt, + job_type=message.type, + duration_seconds=time.perf_counter() - started, + ) + return result + except Exception as error: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute("error.type", type(error).__name__) + signals.event("job.error", error_type=type(error).__name__, attempt=message.attempt) + raise + finally: + signals.worker_busy.add(-1) + + +async def acknowledge(client, message_id): + # Only after result/terminal error or a durable retry intention is committed. + # XDEL follows XACK; no active/unacknowledged work is trimmed. + async with client.pipeline(transaction=True) as transaction: + transaction.xack(STREAM, GROUP, message_id) + transaction.xdel(STREAM, message_id) + await transaction.execute() + + +async def consume(factory, index): + client = broker() + consumer = f"{socket.gethostname()}-{os.getpid()}-{index}" + cursor = "0-0" + try: + while True: + try: + try: + await client.xgroup_create(STREAM, GROUP, id="0", mkstream=True) + except ResponseError as error: + if "BUSYGROUP" not in str(error): + raise + # One uncompleted message per consumer: concurrency=prefetch=N. + claimed = await client.xautoclaim( + STREAM, + GROUP, + consumer, + min_idle_time=1000, + start_id=cursor, + count=1, + ) + cursor, messages = claimed[0], claimed[1] + if not messages: + batches = await client.xreadgroup( + GROUP, consumer, {STREAM: ">"}, count=1, block=1000 + ) + messages = batches[0][1] if batches else [] + for message_id, fields in messages: + try: + message = Envelope.model_validate_json(fields["job"]) + UUID(message.job_id) + UUID(message.attachment_id) + except (ValueError, KeyError): + # A bounded rejection record, never arbitrary payload/credentials. + await client.xadd( + "attachment-rejected-v1", + {"message_id": message_id, "error_code": "invalid_envelope"}, + maxlen=1000, + ) + await acknowledge(client, message_id) + continue + try: + done = await process_job(factory, message) + except PermanentError: + await client.xadd( + "attachment-rejected-v1", + {"message_id": message_id, "error_code": "envelope_mismatch"}, + maxlen=1000, + ) + done = True + if done: + await acknowledge(client, message_id) + except Exception as error: + logger.warning("consumer unavailable: %s", type(error).__name__) + await asyncio.sleep(1) + finally: + await client.aclose() + + +async def main(): + signals.setup("messenger-worker") + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + await asyncio.gather(*(consume(factory, i) for i in range(settings.worker_concurrency))) + finally: + await engine.dispose() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) diff --git a/src/service/__init__.py b/src/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/service/attachment.py b/src/service/attachment.py new file mode 100644 index 0000000..b1d22c8 --- /dev/null +++ b/src/service/attachment.py @@ -0,0 +1,209 @@ +"""Validate originals without re-encoding; keep only metadata in PostgreSQL.""" + +import asyncio +import hashlib +import logging +import unicodedata +import warnings +from datetime import datetime, timezone +from tempfile import TemporaryFile +from uuid import UUID, uuid4 + +from anyio import CapacityLimiter, WouldBlock, to_thread +from fastapi import Request +from PIL import Image, UnidentifiedImageError +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.requests import ClientDisconnect + +from src.core.db.models import Attachment, AttachmentJob, ChatMember, UploadIntent +from src.core.errors import ApiError +from src.core.settings import settings +from src.core.storage import STORAGE_ERRORS, get_storage, unavailable +from src.core.upload_coordination import lock_object +from src.observability import signals + +logger = logging.getLogger(__name__) +_upload_limiter = CapacityLimiter(settings.upload_concurrency) +_decode_limiter = CapacityLimiter(1) +# The explicit dimensions check below is authoritative; convert Pillow warnings +# to errors as an additional guard before decoding malicious image headers. +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or any( + c in "/\\" or unicodedata.category(c) == "Cc" or 0xD800 <= ord(c) <= 0xDFFF + for c in filename + ): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + + +def validate_image(file, content_type: str) -> None: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + file.seek(0) + with Image.open(file) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type != content_type: + raise ApiError(415, "unsupported_media_type", "Тип изображения не совпадает") + if image.width * image.height > settings.max_image_pixels: + raise ApiError(413, "payload_too_large", "Превышен размер изображения") + image.verify() + file.seek(0) + with Image.open(file) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error: + raise ApiError(413, "payload_too_large", "Превышен размер изображения") from error + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError) as error: + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error + finally: + file.seek(0) + + +async def require_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> None: + member = await session.scalar( + select(ChatMember.user_id).where( + ChatMember.chat_id == chat_id, ChatMember.user_id == user_id + ) + ) + if member is None: + raise ApiError(404, "not_found", "Объект не найден") + + +class AttachmentService: + def __init__(self, session: AsyncSession): + self.session = session + + async def get(self, attachment_id: UUID, user_id: UUID) -> Attachment: + attachment = await self.session.scalar( + select(Attachment) + .join(ChatMember, ChatMember.chat_id == Attachment.chat_id) + .where(Attachment.id == attachment_id, ChatMember.user_id == user_id) + ) + if attachment is None: + raise ApiError(404, "not_found", "Вложение не найдено") + if attachment.storage_missing_since is not None and attachment.status == "ready": + raise ApiError(503, "unavailable", "Оригинал вложения временно недоступен") + return attachment + + async def upload( + self, request: Request, chat_id: UUID, user_id: UUID, filename: str + ) -> Attachment: + await require_chat_member(self.session, chat_id, user_id) + validate_filename(filename) + content_type = request.headers.get("content-type", "").strip().lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Требуется PNG или JPEG") + try: + length = int(request.headers.get("content-length", "0")) + if length < 0: + raise ValueError + except ValueError: + raise ApiError(400, "invalid_request", "Некорректная длина тела") from None + if length > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Превышен размер файла") + try: + _upload_limiter.acquire_nowait() + except WouldBlock: + raise ApiError(503, "unavailable", "Слишком много загрузок") from None + try: + return await self._upload(request, chat_id, user_id, filename, content_type) + finally: + _upload_limiter.release() + + async def _upload(self, request, chat_id, user_id, filename, content_type): + size = 0 + digest = hashlib.sha256() + with TemporaryFile() as file: + try: + async with asyncio.timeout(settings.upload_timeout_seconds): + async for chunk in request.stream(): + size += len(chunk) + if size > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Превышен размер файла") + digest.update(chunk) + await to_thread.run_sync(file.write, chunk) + except ClientDisconnect: + raise ApiError(400, "invalid_request", "Передача файла прервана") from None + except TimeoutError: + raise ApiError(408, "invalid_request", "Истекло время приёма файла") from None + await to_thread.run_sync(validate_image, file, content_type, limiter=_decode_limiter) + attachment_id = uuid4() + object_key = f"staging/{attachment_id}" + # Durable registration is independent of the HTTP transaction. If the + # process dies after PUT, the reconciler still knows this key. + async with AsyncSession(bind=self.session.bind) as registration: + async with registration.begin(): + await lock_object(registration, object_key, wait=True) + registration.add( + UploadIntent( + object_key=object_key, + state="pending", + created_at=await registration.scalar(select(func.clock_timestamp())), + ) + ) + await lock_object(self.session, object_key, wait=True) + intent = await self.session.get(UploadIntent, object_key) + if intent is None or intent.state != "pending": + raise ApiError(503, "unavailable", "Попытка загрузки уже закрыта") + storage = get_storage() + try: + await to_thread.run_sync( + lambda: storage.put_object( + settings.s3_bucket, + object_key, + file, + size, + content_type=content_type, + num_parallel_uploads=1, + ) + ) + except STORAGE_ERRORS as error: + raise unavailable(error) from error + now = datetime.now(timezone.utc) + attachment = Attachment( + id=attachment_id, + chat_id=chat_id, + owner_id=user_id, + filename=filename, + object_key=f"originals/{attachment_id}", + staging_key=object_key, + status="queued", + variants=[], + content_type=content_type, + size_bytes=size, + sha256=digest.hexdigest(), + created_at=now, + updated_at=now, + ) + try: + self.session.add(attachment) + await self.session.flush() + self.session.add( + AttachmentJob( + id=uuid4(), + attachment_id=attachment_id, + pipeline_version=1, + traceparent=signals.carrier().get("traceparent"), + tracestate=signals.carrier().get("tracestate"), + state="queued", + attempt=1, + cycle_attempt=1, + created_at=now, + updated_at=now, + next_run_at=now, + ) + ) + # Intent completion and Attachment publication are one DB commit. + # Keep the advisory lock from before PUT through this commit. + intent.state = "committed" + await self.session.commit() + except Exception: + # A failed COMMIT can have an unknown outcome. Keep the object rather + # than deleting bytes that a committed Attachment might reference. + logger.warning("Метаданные загрузки не подтверждены; требуется сверка S3/БД") + raise + signals.event("attachment.accepted", job_type="attachment.process.v1") + return attachment diff --git a/src/service/auth.py b/src/service/auth.py new file mode 100644 index 0000000..425f285 --- /dev/null +++ b/src/service/auth.py @@ -0,0 +1,74 @@ +import logging +import secrets +from datetime import datetime, timedelta, timezone + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core import session_store +from src.core.db.repo import auth as auth_repo +from src.core.db.repo import user as user_repo +from src.core.security import hash_password, hash_token, verify_password_async +from src.core.settings import settings +from src.service.dto import AuthSessionDTO, StoredSessionDTO, UserDTO +from src.service.exceptions import InvalidCredentialsException, InvalidSessionException + +logger = logging.getLogger(__name__) +# Для неизвестного логина тоже выполняем Argon2, чтобы не отвечать заметно быстрее. +_DUMMY_PASSWORD_HASH = hash_password("dummy-password-for-timing") + + +class AuthService: + def __init__( + self, + db_session: AsyncSession, + *, + session_ttl_seconds: int, + ) -> None: + self._db_session = db_session + self._session_ttl = timedelta(seconds=session_ttl_seconds) + + async def login(self, *, username: str, password: str) -> AuthSessionDTO: + credentials = await auth_repo.fetch_user_credentials( + session=self._db_session, + username=username, + ) + password_hash = credentials.password_hash if credentials else _DUMMY_PASSWORD_HASH + password_valid = await verify_password_async(password, password_hash) + if credentials is None or not password_valid: + logger.warning("Вход отклонён: неверный логин или пароль") + raise InvalidCredentialsException("Неверный логин или пароль") + + user = credentials.user + token = secrets.token_urlsafe(settings.session_token_bytes) + expires_at = datetime.now(timezone.utc) + self._session_ttl + await session_store.create_session( + token_hash=hash_token(token), + session=StoredSessionDTO(user_id=user.id, expires_at=expires_at), + ) + logger.info("Сессия создана: пользователь=%s", user.id) + return AuthSessionDTO( + token=token, + expires_at=expires_at, + user=user, + ) + + async def get_user(self, *, token: str) -> UserDTO: + session = await session_store.fetch_session(token_hash=hash_token(token)) + if session is None: + logger.info("Доступ отклонён: сессия отсутствует или истекла") + raise InvalidSessionException("Нет действующей сессии") + + user = await user_repo.fetch_user_by_id( + session=self._db_session, + user_id=session.user_id, + ) + if user is None: + logger.warning("Доступ отклонён: пользователь сессии не найден") + raise InvalidSessionException("Нет действующей сессии") + + logger.debug("Сессия подтверждена: пользователь=%s", user.id) + return user + + async def logout(self, *, token: str) -> None: + await session_store.delete_session(token_hash=hash_token(token)) + logger.info("Сессия завершена") diff --git a/src/service/chat.py b/src/service/chat.py new file mode 100644 index 0000000..97ce519 --- /dev/null +++ b/src/service/chat.py @@ -0,0 +1,61 @@ +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.repo import chat as chat_repo +from src.core.db.repo import user as user_repo +from src.service.dto import ChatDTO +from src.service.exceptions import ChatNotFoundException, InvalidChatMembersException + + +class ChatService: + def __init__(self, db_session: AsyncSession) -> None: + self._db_session = db_session + + async def create_chat( + self, + *, + title: str, + member_ids: list[UUID], + current_user_id: UUID, + ) -> ChatDTO: + if current_user_id not in member_ids: + raise InvalidChatMembersException("Создатель должен быть в списке member_ids") + if not await user_repo.users_exist( + session=self._db_session, + user_ids=member_ids, + ): + raise InvalidChatMembersException("Указан неизвестный пользователь") + + return await chat_repo.create_chat( + session=self._db_session, + chat_id=uuid4(), + title=title, + member_ids=member_ids, + created_at=datetime.now(timezone.utc), + ) + + async def get_chat(self, *, chat_id: UUID, current_user_id: UUID) -> ChatDTO: + chat = await chat_repo.fetch_chat_by_id( + session=self._db_session, + chat_id=chat_id, + ) + self._check_access(chat=chat, user_id=current_user_id) + return chat + + async def get_chats( + self, + *, + current_user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, + ) -> list[ChatDTO]: + return await chat_repo.fetch_chats( + session=self._db_session, user_id=current_user_id, limit=limit, after=after + ) + + @staticmethod + def _check_access(*, chat: ChatDTO | None, user_id: UUID) -> None: + if chat is None or user_id not in chat.member_ids: + raise ChatNotFoundException("Чат не найден") diff --git a/src/service/dto.py b/src/service/dto.py new file mode 100644 index 0000000..ed799ff --- /dev/null +++ b/src/service/dto.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass, field +from datetime import datetime +from typing import Literal +from uuid import UUID + + +@dataclass(frozen=True, slots=True) +class UserDTO: + id: UUID + username: str + display_name: str + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class ChatDTO: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class MessageDTO: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + +@dataclass(frozen=True, slots=True) +class AuthSessionDTO: + token: str = field(repr=False) + expires_at: datetime + user: UserDTO + token_type: Literal["Bearer"] = "Bearer" + + +@dataclass(frozen=True, slots=True) +class StoredSessionDTO: + user_id: UUID + expires_at: datetime + + +@dataclass(frozen=True, slots=True) +class UserCredentialsDTO: + user: UserDTO + password_hash: str = field(repr=False) diff --git a/src/service/exceptions.py b/src/service/exceptions.py new file mode 100644 index 0000000..4c682db --- /dev/null +++ b/src/service/exceptions.py @@ -0,0 +1,30 @@ +class IdempotencyConflictException(Exception): + pass + + +class ServiceException(Exception): + pass + + +class UserAlreadyExistsException(ServiceException): + pass + + +class UserNotFoundException(ServiceException): + pass + + +class InvalidChatMembersException(ServiceException): + pass + + +class ChatNotFoundException(ServiceException): + pass + + +class InvalidCredentialsException(ServiceException): + pass + + +class InvalidSessionException(ServiceException): + pass diff --git a/src/service/message.py b/src/service/message.py new file mode 100644 index 0000000..e10ffb0 --- /dev/null +++ b/src/service/message.py @@ -0,0 +1,64 @@ +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.repo import chat as chat_repo +from src.core.db.repo import message as message_repo +from src.service.dto import ChatDTO, MessageDTO +from src.service.exceptions import ChatNotFoundException + + +class MessageService: + def __init__(self, db_session: AsyncSession) -> None: + self._db_session = db_session + + async def create_message( + self, + *, + chat_id: UUID, + sender_id: UUID, + text: str, + idempotency_key: str, + attachment_ids: list[UUID] | None = None, + ) -> MessageDTO: + chat = await chat_repo.fetch_chat_by_id( + session=self._db_session, + chat_id=chat_id, + ) + self._check_access(chat=chat, user_id=sender_id) + return await message_repo.create_idempotent_message( + session=self._db_session, + message_id=uuid4(), + idempotency_key=idempotency_key, + chat_id=chat_id, + sender_id=sender_id, + text=text, + attachment_ids=attachment_ids or [], + created_at=datetime.now(timezone.utc), + ) + + async def get_messages( + self, + *, + chat_id: UUID, + current_user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, + ) -> list[MessageDTO]: + chat = await chat_repo.fetch_chat_by_id( + session=self._db_session, + chat_id=chat_id, + ) + self._check_access(chat=chat, user_id=current_user_id) + return await message_repo.fetch_messages_by_chat_id( + session=self._db_session, + chat_id=chat_id, + limit=limit, + after=after, + ) + + @staticmethod + def _check_access(*, chat: ChatDTO | None, user_id: UUID) -> None: + if chat is None or user_id not in chat.member_ids: + raise ChatNotFoundException("Чат не найден") diff --git a/src/service/user.py b/src/service/user.py new file mode 100644 index 0000000..05b6882 --- /dev/null +++ b/src/service/user.py @@ -0,0 +1,63 @@ +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.repo import user as user_repo +from src.core.security import hash_password_async +from src.service.dto import UserDTO +from src.service.exceptions import UserAlreadyExistsException, UserNotFoundException + + +class UserService: + def __init__(self, db_session: AsyncSession) -> None: + self._db_session = db_session + + async def create_user(self, *, username: str, display_name: str, password: str) -> UserDTO: + password_hash = await hash_password_async(password) + try: + return await user_repo.create_user( + session=self._db_session, + user_id=uuid4(), + username=username, + display_name=display_name, + created_at=datetime.now(timezone.utc), + password_hash=password_hash, + ) + except IntegrityError as error: + if getattr(error.orig, "sqlstate", None) != "23505": + raise + raise UserAlreadyExistsException("Пользователь с таким именем уже существует") from None + + async def create_admin( + self, + *, + username: str, + display_name: str, + password: str, + ) -> UserDTO | None: + password_hash = await hash_password_async(password) + user = await user_repo.create_admin( + session=self._db_session, + user_id=uuid4(), + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=datetime.now(timezone.utc), + ) + if user is None and not await user_repo.admin_exists( + session=self._db_session, + username=username, + ): + raise UserAlreadyExistsException("Имя администратора занято обычным пользователем") + return user + + async def get_user_by_id(self, *, user_id: UUID) -> UserDTO: + user = await user_repo.fetch_user_by_id( + session=self._db_session, + user_id=user_id, + ) + if user is None: + raise UserNotFoundException("Пользователь не найден") + return user diff --git a/tests/api_regressions.py b/tests/api_regressions.py new file mode 100644 index 0000000..667bb92 --- /dev/null +++ b/tests/api_regressions.py @@ -0,0 +1,199 @@ +"""Replay concrete reviewer cases through real HTTP and controlled dependency freezes.""" +import io +import json +import struct +import subprocess +import time +import zlib +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from client import Client, arguments, check, run + +ROOT=Path(__file__).resolve().parents[1] + + +def compose(*args): + return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip() + + +def png(width,height): + # Complete PNG, including all scanlines and CRCs, without allocating a bitmap. + def chunk(kind,data): + return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)) + compressor=zlib.compressobj() + packed=io.BytesIO() + row=b'\0'*(width+1) + for _ in range(height): packed.write(compressor.compress(row)) + packed.write(compressor.flush()) + return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,0,0,0,0)) + +chunk(b'IDAT',packed.getvalue())+chunk(b'IEND',b'')) + + +DIRECT='''import json,sys,time,urllib.request,urllib.error +p=json.load(sys.stdin);h={} +if p.get('token'):h['Authorization']='Bearer '+p['token'] +r=urllib.request.Request('http://127.0.0.1:8080'+p['path'],headers=h) +t=time.monotonic() +try:r=urllib.request.urlopen(r,timeout=10) +except urllib.error.HTTPError as e:r=e +with r: + data=json.load(r) + if r.code>=400: + assert data['error']['request_id']==r.headers['X-Request-Id'] + assert data['error']['code']=='unavailable' + print(json.dumps({'status':r.code,'seconds':round(time.monotonic()-t,4), + 'request_id_present':bool(r.headers.get('X-Request-Id')),'instance_id_present':bool(r.headers.get('X-Instance-Id'))})) +''' +LOCKER='''import asyncio,sys,asyncpg +from src.core.settings import settings +async def f(): + c=await asyncpg.connect(settings.database_url.get_secret_value()) + await c.execute('BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE') + print('LOCKED',flush=True) + await asyncio.to_thread(sys.stdin.readline) + await c.execute('ROLLBACK');await c.close() +asyncio.run(f()) +''' +PENDING='''import asyncio,asyncpg +from src.core.settings import settings +async def f(): + c=await asyncpg.connect(settings.database_url.get_secret_value()) + print(await c.fetchval("SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE '%FROM messages%'")) + await c.close() +asyncio.run(f()) +''' + + +def main(): + args=arguments('API reviewer regression scenarios') + c=Client(args.base_url,args.ca_file,timeout=15) + alice,bob,_,chat=c.bootstrap() + history='/api/v1/chats/'+chat['id']+'/messages' + c.message(alice,chat,'Контроль до отказов') + results=[] + for name in ('review123\n','review123\r','review123\x00','review123\t'): + c.call('POST','/api/v1/users',400,body={'username':name,'display_name':'Review','password':'valid-password-123'}) + results.append({'case':'username_control_characters','result':'PASS','status':400}) + data=(ROOT/'tests/fixtures/sample.png').read_bytes() + upload='/api/v1/chats/'+chat['id']+'/attachments' + for label,filename,payload,mime,status in [ + ('truncated_png','sample.png',data[:-12],'image/png',415), + ('header_only_png','sample.png',data[:33],'image/png',415), + ('mismatched_mime','sample.png',data,'image/jpeg',415), + ('DEL_filename','a%7F.png',data,'image/png',400), + ('missing_filename',None,data,'image/png',400), + ]: + c.call('POST',upload+('' if filename is None else '?filename='+filename),status, + actor=alice,raw=payload,headers={'Content-Type':mime}) + results.append({'case':label,'result':'PASS','status':status}) + for width,height,status in ((8000,5000,202),(8000,5001,413),(20000,10000,413)): + payload=png(width,height) + reply=c.call('POST',upload+'?filename=dimensions.png',status,actor=alice,raw=payload, + headers={'Content-Type':'image/png'},schema='Attachment' if status==202 else None) + if status==202: + c.wait_ready(alice, reply['json']) + check(c.call('GET','/api/v1/attachments/'+reply['json']['id']+'/content',actor=bob)['bytes']==payload, + '40 MP original changed') + results.append({'case':'pixels','dimensions':[width,height],'result':'PASS','status':status}) + api=compose('ps','-q','api').split()[0] + # Actual application credentials must not permit creating unrelated buckets. + code='''from src.core.storage import get_storage +from minio.error import S3Error +import uuid +s=get_storage();b='forbidden-'+uuid.uuid4().hex +try:s.make_bucket(b) +except S3Error as e:assert e.code=='AccessDenied';print('PASS') +else:s.remove_bucket(b);raise AssertionError('API can create foreign buckets') +''' + check(subprocess.check_output(['docker','exec',api,'python','-c',code],text=True).strip()=='PASS', + 'API has excessive S3 rights') + results.append({'case':'API_cannot_create_foreign_bucket','result':'PASS'}) + def direct(path,token=None): + reply=subprocess.run(['docker','exec','-i',api,'python','-c',DIRECT], + input=json.dumps({'path':path,'token':token}),text=True,capture_output=True,timeout=13,check=True) + return json.loads(reply.stdout) + transfer_code="""import asyncio,http.client,json,sys,time +from sqlalchemy import func,select +from sqlalchemy.ext.asyncio import async_sessionmaker +from src.core.db.session import create_engine +from src.core.db.models import Attachment,UploadIntent +p=json.load(sys.stdin) +async def counts(): + e=create_engine();f=async_sessionmaker(e) + try: + async with f() as s: + return [await s.scalar(select(func.count()).select_from(m)) for m in (Attachment,UploadIntent)] + finally:await e.dispose() +before=asyncio.run(counts()) +c=http.client.HTTPConnection('127.0.0.1',8080,timeout=8) +c.putrequest('POST',p['path']) +c.putheader('Authorization','Bearer '+p['token']) +c.putheader('Content-Type','image/png');c.putheader('Content-Length','1000');c.endheaders() +c.send(b'\\x89PNG\\r\\n\\x1a\\nshort');c.close();time.sleep(.3) +assert before==asyncio.run(counts()),'interrupted client created durable state' +c=http.client.HTTPConnection('127.0.0.1',8080,timeout=8) +c.request('POST',p['path'],body=(b'x'*65536 for _ in range(161)), + headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'},encode_chunked=True) +r=c.getresponse();body=json.loads(r.read()) +assert r.status==413 and body['error']['code']=='payload_too_large' +assert body['error']['request_id']==r.getheader('X-Request-Id') +c.close();assert before==asyncio.run(counts()),'oversized chunked body created durable state' +print('PASS') +""" + transfer=subprocess.run(['docker','exec','-i',api,'python','-c',transfer_code], + input=json.dumps({'path':upload+'?filename=interrupted.png','token':alice['token']}), + text=True,capture_output=True,timeout=20,check=True) + check(transfer.stdout.strip()=='PASS','transfer limits failed') + results.append({'case':'client_abort_and_oversized_chunked_body','result':'PASS', + 'oversized_status':413,'durable_state_unchanged':True}) + direct(history,alice['token']) + # Start an actual history SELECT, hold it on a table lock, then freeze PG. + locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',LOCKER], + stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + check(locker.stdout.readline().strip()=='LOCKED','test table lock was not acquired') + paused=False + pool=ThreadPoolExecutor(max_workers=2) + query=pool.submit(direct,history,alice['token']) + try: + deadline=time.monotonic()+2 + while True: + pending=int(subprocess.check_output(['docker','exec',api,'python','-c',PENDING],text=True)) + if pending:break + check(time.monotonic()deadline:raise + time.sleep(.2) + results.append({'case':'PostgreSQL_resume_without_API_restart','result':'PASS'}) + print(json.dumps({'result':'PASS','checks':results},ensure_ascii=False,indent=2)) + + +if __name__=='__main__': + run(main) diff --git a/tests/attachment_failover.py b/tests/attachment_failover.py new file mode 100644 index 0000000..94225a3 --- /dev/null +++ b/tests/attachment_failover.py @@ -0,0 +1,45 @@ +"""Stop the exact API replica which accepted the image; preserve original bytes.""" +import os +import subprocess +import time +from pathlib import Path + +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Attachment download after stopping the uploading replica') + c = Client(args.base_url, args.ca_file) + alice, bob, _, chat = c.bootstrap() + content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', + 202, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, + schema='Attachment') + att = uploaded['json'] + c.message(alice, chat, '', attachments=[att['id']]) + c.wait_ready(alice, att) + target = uploaded['headers']['X-Instance-Id'] + subprocess.run(['scripts/stop-one.sh'], check=True, + env=dict(os.environ, TARGET_INSTANCE_ID=target), + stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 15 + try: + while True: + try: + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + break + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(.2) + check(downloaded['bytes'] == content, 'original lost after uploader replica stopped') + check(downloaded['headers']['X-Instance-Id'] != target, 'download used stopped replica') + print('PASS: exact uploading replica stopped; another replica returns identical original') + finally: + # Restore the two-replica development stack for subsequent work. + subprocess.run(['docker', 'compose', 'up', '-d', '--wait', 'api'], check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +if __name__ == '__main__': + run(main) diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 0000000..35ed5cb --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,99 @@ +"""Lab4 black-box checks against the running Compose stack.""" +import hashlib +import io +import struct +import zlib +from concurrent.futures import ThreadPoolExecutor +import uuid +from pathlib import Path + +from PIL import Image +from client import Client, arguments, check, run + + +def main(): + args = arguments('Attachment originals, ACL, validation and message limits') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + + def upload(data=content, name='photo.png', mime='image/png', status=202, actor=alice): + return c.call('POST', path + '/attachments?filename=' + name, status, + actor=actor, raw=data, headers={'Content-Type': mime}, + schema='Attachment' if status == 202 else None)['json'] + + first = upload() + second = upload() # Same display name must not overwrite the first object. + check(first['id'] != second['id'], 'same filename overwrote attachment') + jpg = io.BytesIO() + Image.new('RGB', (31, 17), 'red').save(jpg, format='JPEG') + jpeg = upload(jpg.getvalue(), 'photo.jpg', 'image/jpeg') + for att, original in [(first, content), (second, content), (jpeg, jpg.getvalue())]: + check(att['sha256'] == hashlib.sha256(original).hexdigest(), 'incorrect digest') + att = c.wait_ready(alice, att) + check({v['name'] for v in att['variants']} == {'thumbnail', 'crop', 'optimized'}, 'incorrect ready metadata') + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve) + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + check(downloaded['bytes'] == original, 'original changed') + + ids = [first['id'], second['id'], jpeg['id']] + key = uuid.uuid4().hex + message = c.message(alice, chat, '', key=key, attachments=ids)['json'] + check(message['attachment_ids'] == ids, 'attachment order lost') + check(c.message(alice, chat, '', key=key, attachments=ids)['json'] == message, + 'idempotency replay changed message') + c.call('POST', path + '/messages', 409, actor=alice, + body={'text': '', 'attachment_ids': ids[:1]}, headers={'Idempotency-Key': key}) + page = c.call('GET', path + '/messages', actor=bob, schema='MessagePage')['json'] + check(any(m == message for m in page['items']), 'history lost attachments') + # The contract permits referencing the same accessible chat attachment again. + c.message(bob, chat, 'same image', attachments=[first['id']]) + + for body in ({'text': '', 'attachment_ids': []}, + {'text': '', 'attachment_ids': [first['id']] * 2}, + {'text': '', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}): + c.call('POST', path + '/messages', 400, actor=alice, body=body, + headers={'Idempotency-Key': uuid.uuid4().hex}) + c.call('POST', path + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': [str(uuid.uuid4())]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + other = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, + schema='Chat')['json'] + c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': ids}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + upload(actor=eve, status=404) + upload(name='..%2Fphoto.png', status=400) + upload(name='a%00b.png', status=400) + upload(data=b'not a PNG', status=415) + upload(data=content[:30], status=415) + upload(mime='image/jpeg', status=415) + upload(mime='application/octet-stream', status=415) + # Oversized dimensions with a valid PNG header; no huge bitmap allocation. + header = struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0) + bomb = (content[:8] + struct.pack('>I', len(header)) + b'IHDR' + header + + struct.pack('>I', zlib.crc32(b'IHDR' + header)) + content[33:]) + upload(data=bomb, status=413) + upload(data=b'x' * (10485760 + 1), status=413) + c.call('POST', path + '/attachments?filename=a.png', 401, + raw=content, headers={'Content-Type': 'image/png'}) + + # Exactly 10 distinct attachments must remain valid, including after reading history. + ten = ids + [upload()['id'] for _ in range(7)] + check(len(c.message(alice, chat, '', attachments=ten)['json']['attachment_ids']) == 10, + '10 attachments rejected') + replay_key = uuid.uuid4().hex + with ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(lambda _: Client(args.base_url, args.ca_file).message( + alice, chat, 'Concurrent image', key=replay_key, attachments=ids + ), range(8))) + check(all(r['json'] == responses[0]['json'] for r in responses), + 'concurrent attachment retries differ') + print('PASS: PNG/JPEG bytes, metadata, ACL, chat boundaries, limits and idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 0000000..7e45699 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Конкурентная идемпотентность, область ключей и повторная авторизация.""" + +import json +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor + +from client import Client, arguments, check, run + + +def main(): + args = arguments("Lab3 concurrency and idempotency") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + key = uuid.uuid4().hex + clients = [Client(args.base_url, args.ca_file) for _ in range(16)] + barrier = threading.Barrier(len(clients)) + + def send(local): + barrier.wait() + return local.message(alice, chat, "Одна отправка 👋", key=key) + + with ThreadPoolExecutor(max_workers=len(clients)) as pool: + responses = list(pool.map(send, clients)) + instances = {r["headers"]["X-Instance-Id"] for r in responses} + check(len(instances) >= 2, "Parallel repeats did not reach >=2 replicas") + original = responses[0]["json"] + check(all(r["json"] == original for r in responses), "Concurrent results differ") + messages = c.call("GET", path, actor=bob, schema="MessagePage")["json"]["items"] + check(messages == [original], "Concurrent retries duplicated message") + # Missing attachment_ids and [] are semantically equal; raw JSON ordering too. + replay = c.call( + "POST", + path, + 201, + actor=alice, + raw=b'{"attachment_ids":[],"text":"' + "Одна отправка 👋".encode() + b'"}', + headers={"Idempotency-Key": key, "Content-Type": "application/json"}, + schema="Message", + ) + check(replay["json"] == original, "Equivalent JSON did not replay") + c.call( + "POST", + path, + 409, + actor=alice, + body={"text": "Другой текст"}, + headers={"Idempotency-Key": key}, + ) + other = c.message(bob, chat, "Другой отправитель", key=key)["json"] + check(other["id"] != original["id"], "Key incorrectly shared across users") + chat2 = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой чат", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + check( + c.message(alice, chat2, "Другой чат", key=key)["json"]["id"] != original["id"], + "Key scope includes wrong chat", + ) + for invalid in [None, "", "bad key", "x" * 129, "bad/key"]: + c.call( + "POST", + path, + 400, + actor=alice, + body={"text": "invalid"}, + headers={} if invalid is None else {"Idempotency-Key": invalid}, + ) + # Validation/access failures must not consume a key. + retry_key = uuid.uuid4().hex + c.call( + "POST", path, 400, actor=alice, body={"text": ""}, headers={"Idempotency-Key": retry_key} + ) + c.message(alice, chat, "После 400", key=retry_key) + c.call( + "POST", + path, + 404, + actor=eve, + body={"text": original["text"]}, + headers={"Idempotency-Key": key}, + ) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call( + "POST", + path, + 401, + actor=alice, + body={"text": original["text"]}, + headers={"Idempotency-Key": key}, + ) + for route in ["/health/live", "/health/ready", "/missing"]: + c.call("GET", route, 404 if route == "/missing" else 200) + print( + json.dumps( + { + "parallel_repeats": len(responses), + "instances": sorted(instances), + "messages_created": 1, + "checks": "normalization, conflict, key scope, invalid keys, access, logout, headers", + "result": "PASS", + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/tests/lab5.py b/tests/lab5.py new file mode 100644 index 0000000..800a4fd --- /dev/null +++ b/tests/lab5.py @@ -0,0 +1,291 @@ +"""Lab5 HTTP checks and real, opt-in Compose fault injection on an isolated stand.""" +import hashlib +import io +import json +import os +import subprocess +import tempfile +import time +import uuid +from pathlib import Path + +from PIL import Image +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parents[1] +RESULTS = [] + + +def passed(name, **details): + result = dict(case=name, result='PASS', **details) + RESULTS.append(result) + print(json.dumps(result, ensure_ascii=False), flush=True) + + +def wait(predicate, description, seconds=40): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + result = predicate() + if result: + return result + time.sleep(.25) + raise AssertionError('Timeout: ' + description) + + +def main(): + def configure(parser): + parser.add_argument('--faults', action='store_true') + parser.add_argument('--project', default=os.environ.get('COMPOSE_PROJECT_NAME', '')) + args = arguments(__doc__, configure) + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + base = '/api/v1/attachments/' + + def upload(data, mime='image/png'): + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=image.png', + 202, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment')['json'] + check(att['status'] == 'queued' and att['variants'] == [], '202 must be a queued snapshot') + return att + + def fixture(size, mode='RGB', color='red', fmt='PNG', exif=None): + data = io.BytesIO() + options = {'exif': exif} if exif else {} + Image.new(mode, size, color).save(data, fmt, **options) + return data.getvalue() + + cases = [((320, 200), (256, 160)), ((7, 3), (7, 3)), ((1001, 667), (256, 170)), + ((1600, 1000), (256, 160))] + for size, thumb_size in cases: + data = fixture(size) + att = c.wait_ready(alice, upload(data)) + variants = {v['name']: v for v in att['variants']} + check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'all variants required') + check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed') + for name, item in variants.items(): + response = c.call('GET', base + att['id'] + '/content?variant=' + name, actor=bob) + encoded = response['bytes'] + check(hashlib.sha256(encoded).hexdigest() == item['sha256'], 'variant checksum') + check(len(encoded) == item['size_bytes'], 'variant size') + with Image.open(io.BytesIO(encoded)) as image: + image.load() + check(image.size == (item['width'], item['height']), 'reported dimensions') + check(not image.getexif() and 'icc_profile' not in image.info, 'metadata leaked') + check(image.format == ('WEBP' if name == 'optimized' else 'JPEG'), 'format') + if name == 'thumbnail': check(image.size == thumb_size, 'floor/no-upscale') + if name == 'crop': check(image.size == (128, 128), 'crop dimensions') + if name == 'optimized': + check(max(image.size) <= 1280 and image.width <= size[0] and image.height <= size[1], 'optimized bounds') + c.call('GET', base + att['id'] + '/content?variant=' + name, 404, actor=eve) + passed('variants', original=list(size), thumbnail=list(thumb_size)) + + alpha = upload(fixture((20, 10), 'RGBA', (255, 0, 0, 0))) + alpha = c.wait_ready(alice, alpha) + encoded = c.call('GET', base + alpha['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + with Image.open(io.BytesIO(encoded)) as image: + check(min(image.getpixel((0, 0))) >= 250, 'alpha must be white') + exif = Image.Exif(); exif[274] = 6; exif[270] = 'private metadata' + rotated = c.wait_ready(alice, upload(fixture((200, 100), fmt='JPEG', exif=exif), 'image/jpeg')) + thumbnail = next(v for v in rotated['variants'] if v['name'] == 'thumbnail') + check((thumbnail['width'], thumbnail['height']) == (100, 200), 'EXIF orientation') + stripes = Image.new('RGB', (300, 100), 'red') + stripes.paste('green', (100, 0, 200, 100)); stripes.paste('blue', (200, 0, 300, 100)) + buf = io.BytesIO(); stripes.save(buf, 'PNG') + cropped = c.wait_ready(alice, upload(buf.getvalue())) + data = c.call('GET', base + cropped['id'] + '/content?variant=crop', actor=bob)['bytes'] + with Image.open(io.BytesIO(data)) as image: + red, green, blue = image.getpixel((64, 64)) + check(green > red + 50 and green > blue + 50, 'crop must be centered') + passed('alpha_EXIF_metadata_center_crop') + + if not args.faults: + return + check(bool(args.project) and args.project != ROOT.name, 'Fault tests require an explicitly named isolated project') + override = Path(tempfile.mkdtemp(prefix='lab5-faults-')) / 'override.yaml' + override.write_text('''services: + worker: + environment: &faults + PIPELINE_FAULT_INJECTION: "true" + JOB_LEASE_SECONDS: "5" + JOB_MAX_ATTEMPTS: "3" + JOB_TIMEOUT_SECONDS: "1" + JOB_RETRY_BASE_SECONDS: "0.5" + JOB_RETRY_CAP_SECONDS: "2" + dispatcher: + environment: *faults +''') + command = ['docker', 'compose', '--env-file', os.environ.get('ENV_FILE', '.env'), '-p', args.project, '-f', str(ROOT / 'compose.yaml'), '-f', str(override)] + + def compose(*parts, input=None, expected=0): + result = subprocess.run(command + list(parts), input=input, text=True, capture_output=True, timeout=100) + check(result.returncode == expected, 'Compose action failed: ' + ' '.join(parts[:3]) + ' ' + result.stderr[-400:]) + return result.stdout.strip() + + def sql(query): + return compose('exec', '-T', 'postgres', 'sh', '-c', + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1', input=query) + + def job(att): + return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='" + att['id'] + "';")) + + def set_fault(att, value): + sql("UPDATE attachment_jobs SET fault=" + ("'" + value + "'" if value else 'NULL') + " WHERE attachment_id='" + att['id'] + "';") + + def exited(service, code): + ids = compose('ps', '-aq', service).split() + if not ids: return False + inspect = subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', ids[0]], text=True) + state = json.loads(inspect) + return state['Status'] == 'exited' and state['ExitCode'] == code + + def pending(): + return int(compose('exec', '-T', 'broker', 'redis-cli', 'XPENDING', 'attachment-jobs-v1', 'image-workers-v1').splitlines()[0]) + + def objects(att): + code = '''import json,sys +from src.core.storage import make_storage +from src.core.settings import settings +s=make_storage(role='worker');i=sys.argv[1] +print(json.dumps({o.object_name:o.etag for prefix in ['originals/'+i,'variants/'+i+'/'] for o in s.list_objects(settings.s3_bucket,prefix=prefix,recursive=True)}))''' + return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code, att['id'])) + + def inject(att): + j = job(att) + payload = dict(schema_version=1, job_id=j['id'], type='attachment.process.v1', attachment_id=att['id'], + pipeline_version=1, attempt=j['attempt'], created_at=j['created_at']) + compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'job', json.dumps(payload)) + + try: + compose('up', '-d', '--no-deps', '--force-recreate', 'worker', 'dispatcher') + compose('stop', 'worker') + att, data = c.attachment(alice, chat) + c.message(alice, chat, '', attachments=[att['id']]) + c.call('GET', base + att['id'] + '/content', 409, actor=bob) + c.message(alice, chat, 'API работает без worker') + wait(lambda: job(att)['published_at'], 'publish while worker stopped') + check(int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) > 0, 'broker must contain work') + compose('restart', 'api', 'broker') + def ready(): + try: + return c.call('GET', '/health/ready', [200, 503])['status'] == 200 + except Failure: + return False + wait(ready, 'API ready') + compose('start', 'worker') + c.wait_ready(alice, att) + check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'restart lost original') + passed('stopped_worker_API_broker_restart', attachment_id=att['id']) + + compose('stop', 'broker') + unavailable = upload(fixture((324, 204))) + check(c.call('GET', '/health/ready')['status'] == 200, 'broker outage must not block API') + check(job(unavailable)['published_at'] is None, 'unsent job must remain in DB') + compose('start', 'broker') + c.wait_ready(alice, unavailable) + passed('broker_unavailable_outbox_recovered') + + # COMMIT is already durable; kill the actual relay before and after XADD. + for fault, code in [('before_publish', 81), ('after_publish', 82)]: + compose('stop', 'dispatcher', 'worker') + att = upload(fixture((321, 201))) + set_fault(att, fault) + check(job(att)['published_at'] is None, 'must have unsent outbox') + compose('start', 'dispatcher') + wait(lambda: exited('dispatcher', code), fault) + check(job(att)['published_at'] is None, 'relay crash must roll back sent flag') + # Dispatcher is stopped; clear through postgres, then restore the same job. + set_fault(att, None) + compose('start', 'dispatcher', 'worker') + c.wait_ready(alice, att) + passed(fault + '_recovered', job_id=job(att)['id']) + + for fault, code in [('after_objects', 83), ('after_commit', 84)]: + compose('stop', 'worker') + att = upload(fixture((322, 202))) + set_fault(att, fault) + compose('start', 'worker') + wait(lambda: exited('worker', code), fault) + before = objects(att) + check(len(before) == 4, 'one original and three objects must exist') + j = job(att) + if fault == 'after_objects': + check(j['state'] == 'processing', 'crash must leave a processing lease') + else: + check(j['state'] == 'ready', 'ready commit must precede ack') + check(pending() > 0, 'crash must leave an unacked delivery') + set_fault(att, None) + compose('start', 'worker') + c.wait_ready(alice, att) + wait(lambda: pending() == 0, 'pending delivery recovery') + check(objects(att) == before, 'redelivery changed/duplicated objects') + stable = job(att) + inject(att); inject(att) + wait(lambda: pending() == 0 and int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) == 0, 'duplicate ack') + check(job(att)['attempt'] == stable['attempt'], 'ready duplicate must not start a new attempt') + check(objects(att) == before, 'ready duplicate changed output') + passed(fault + '_redelivery_idempotent', attempt=stable['attempt'], objects=len(before)) + + for fault in ['temporary', 'permanent', 'timeout']: + compose('stop', 'worker') + att = upload(fixture((323, 203))) + set_fault(att, fault) + compose('start', 'worker') + failed = wait(lambda: (a if (a := c.call('GET', base + att['id'], actor=alice, schema='Attachment')['json'])['status'] == 'failed' else None), 'terminal failed') + j = job(att) + check(j['cycle_attempt'] == (1 if fault == 'permanent' else 3), 'retry budget/classification') + check(failed['error_code'] == ('invalid_image' if fault == 'permanent' else 'retry_exhausted'), 'safe terminal code') + c.call('GET', base + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text':'', 'attachment_ids':[att['id']]}, headers={'Idempotency-Key':uuid.uuid4().hex}) + registry = json.loads(compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'failed')) + check(any(x['job_id'] == j['id'] for x in registry), 'failed registry missing') + set_fault(att, None) + compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'dependency restored') + c.wait_ready(alice, att) + check(sql("SELECT count(*) FROM job_replays WHERE job_id='" + j['id'] + "';") == '1', 'audit missing') + compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'duplicate replay', expected=1) + passed(fault + '_failed_audited_replay', attempts_before_replay=j['attempt']) + + wait(lambda: sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;") == '0', 'ready staging cleanup') + passed('staging_cleanup_after_ready') + + compose('stop', 'worker') + retained = upload(fixture((325, 205))) + set_fault(retained, 'permanent') + compose('start', 'worker') + wait(lambda: job(retained)['state'] == 'failed', 'failed retention fixture') + check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'fresh failed staging must remain') + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + retained['id'] + "';") + wait(lambda: sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'failed retention cleanup') + compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', job(retained)['id'], '--operator', 'lab5-test', '--reason', 'expired source', expected=1) + passed('failed_retention_and_expired_replay_rejected') + + compose('stop', 'worker') + active = upload(fixture((326, 206))) + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + active['id'] + "';") + orphan = 'staging/' + str(uuid.uuid4()) + script = '''import io,sys +from src.core.storage import make_storage +from src.core.settings import settings +make_storage(role='worker').put_object(settings.s3_bucket,sys.argv[1],io.BytesIO(b'orphan'),6) +''' + compose('exec', '-T', 'dispatcher', 'python', '-c', script, orphan) + sql("INSERT INTO upload_intents(object_key,state,created_at) VALUES ('" + orphan + "','pending',clock_timestamp()-interval '8 days');") + wait(lambda: sql("SELECT state FROM upload_intents WHERE object_key='" + orphan + "';") == 'abandoned', 'orphan staging cleanup') + check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + active['id'] + "';") == 't', 'cleanup must protect queued work') + compose('start', 'worker') + c.wait_ready(alice, active) + passed('abandoned_staging_cleaned_active_job_protected') + + finally: + # Disable every injection and restore the ordinary configuration even on failure. + try: + sql('UPDATE attachment_jobs SET fault=NULL WHERE fault IS NOT NULL;') + finally: + subprocess.run(['docker','compose','--env-file',os.environ.get('ENV_FILE','.env'),'-p',args.project,'up','-d','--no-deps','--force-recreate','worker','dispatcher'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=100, check=True) + override.unlink(missing_ok=True) + override.parent.rmdir() + + +if __name__ == '__main__': + run(main) diff --git a/tests/lab5_limits.py b/tests/lab5_limits.py new file mode 100644 index 0000000..1c55156 --- /dev/null +++ b/tests/lab5_limits.py @@ -0,0 +1,44 @@ +"""Run in Linux worker: actual decoder failures and configured resource ceilings.""" +import io +import json +import os +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +from PIL import Image + +from src.core.settings import settings + + +def main(): + with TemporaryDirectory() as folder: + root = Path(folder) + source = root / 'input' + def process(data, max_pixels=40000000, memory=768): + source.write_bytes(data) + return subprocess.run([sys.executable,'-m','src.pipeline.images',str(source),str(root), + str(max_pixels),str(memory),'5'],capture_output=True,timeout=10, + env=dict(os.environ, PIPELINE_TEST_IMAGE_DELAY='0')).returncode + buffer = io.BytesIO() + Image.new('RGB',(1000,1000),'red').save(buffer,'PNG') + data = buffer.getvalue() + assert process(b'not an image') == 2 + assert process(data[:33]) == 2 + assert process(data, max_pixels=100) == 2 + assert process(data, memory=1) != 0 + assert process(data) == 0 + # Non-square integer-ratio rounding and independent process are exercised + # by HTTP tests. These values prove the actual stand configuration too. + assert settings.worker_memory_mb == 768 + assert settings.worker_concurrency == 1 + print(json.dumps(dict(result='PASS', invalid_image_rejected=True, + truncated_image_rejected=True, decoded_pixel_limit=True, + address_space_limit=True, subprocess_memory_mb=settings.worker_memory_mb, + wall_timeout_seconds=settings.job_timeout_seconds, + concurrency=settings.worker_concurrency, prefetch_per_consumer=1))) + + +if __name__ == '__main__': + main() diff --git a/tests/lab5_migration.py b/tests/lab5_migration.py new file mode 100644 index 0000000..d173c71 --- /dev/null +++ b/tests/lab5_migration.py @@ -0,0 +1,90 @@ +"""Run inside worker: migrate a real lab4 database and process its existing file.""" +import asyncio +import hashlib +import io +import json +import os +import subprocess +from urllib.parse import urlsplit, urlunsplit +from uuid import uuid4 + +import asyncpg +from PIL import Image +from pydantic import SecretStr +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.pipeline.dispatcher import cleanup +from src.pipeline.worker import Envelope, process_job + + +async def main(): + previous = settings.database_url.get_secret_value() + database = 'migration_' + uuid4().hex + parts = urlsplit(previous) + url = urlunsplit(parts._replace(path='/' + database)) + control = await asyncpg.connect(previous) + storage = make_storage(role='worker') + attachment_id, user_id, chat_id = uuid4(), uuid4(), uuid4() + key = 'originals/' + str(attachment_id) + engine = None + try: + await control.execute('CREATE DATABASE ' + database) + env = dict(os.environ, DATABASE_URL=url) + def migrate(target): + result = subprocess.run(['alembic', 'upgrade', target], env=env, capture_output=True) + assert result.returncode == 0, 'migration failed: ' + result.stderr.decode()[-500:] + await asyncio.to_thread(migrate, '008') + buffer = io.BytesIO() + Image.new('RGB', (320, 200), 'orange').save(buffer, 'PNG') + data = buffer.getvalue() + await asyncio.to_thread(storage.put_object, settings.s3_bucket, key, io.BytesIO(data), len(data), 'image/png') + connection = await asyncpg.connect(url) + try: + await connection.execute("INSERT INTO users (id,username,display_name,password_hash,created_at) VALUES ($1,'migration_user','Migration','unused',now())", user_id) + await connection.execute("INSERT INTO chats (id,title,created_at) VALUES ($1,'Migration',now())", chat_id) + await connection.execute("INSERT INTO chat_members (chat_id,user_id) VALUES ($1,$2)", chat_id, user_id) + await connection.execute("""INSERT INTO attachments + (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) + VALUES ($1,$2,$3,'legacy.png',$4,'image/png',$5,$6,now(),now())""", + attachment_id, chat_id, user_id, key, len(data), hashlib.sha256(data).hexdigest()) + finally: + await connection.close() + await asyncio.to_thread(migrate, 'head') + settings.database_url = SecretStr(url) + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + async with factory() as session: + attachment = await session.get(Attachment, attachment_id) + job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == attachment_id)) + assert attachment.status == 'queued' and attachment.staging_key == key + message = Envelope(schema_version=1,job_id=str(job.id),type='attachment.process.v1', + attachment_id=str(attachment_id),pipeline_version=1,attempt=1,created_at=job.created_at.isoformat()) + assert await process_job(factory, message) + await cleanup(factory) + async with factory() as session: + attachment = await session.get(Attachment, attachment_id) + assert attachment.status == 'ready' and len(attachment.variants) == 3 + assert attachment.staging_key is None + response = storage.get_object(settings.s3_bucket, key) + try: + assert response.read() == data, 'cleanup removed/replaced legacy original' + finally: + response.close(); response.release_conn() + print(json.dumps(dict(result='PASS', migration='008 -> 009', legacy_original_unchanged=True, + variants=3, staging_cleanup_safe=True))) + finally: + settings.database_url = SecretStr(previous) + if engine: + await engine.dispose() + await control.execute('DROP DATABASE IF EXISTS ' + database + ' WITH (FORCE)') + await control.close() + for object_key in [key] + ['variants/' + str(attachment_id) + '/v1/' + name for name in ('thumbnail','crop','optimized')]: + await asyncio.to_thread(storage.remove_object, settings.s3_bucket, object_key) + + +asyncio.run(main()) diff --git a/tests/observability.py b/tests/observability.py new file mode 100644 index 0000000..0cf56c3 --- /dev/null +++ b/tests/observability.py @@ -0,0 +1,207 @@ +"""Short real-backend check; optional bounded incidents for the grade-4 evidence.""" +import json +import os +import subprocess +import time +import urllib.parse +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from client import Client, Failure, arguments, check, run + +ROOT=Path(__file__).resolve().parents[1] +EVIDENCE=ROOT/'evidence' + + +def get(url): + request=urllib.request.Request(url,headers={'Accept':'application/json'}) + with urllib.request.urlopen(request,timeout=5) as response: + return json.load(response) + + +def wait(fn, label, seconds=35): + deadline=time.monotonic()+seconds + error=None + while time.monotonic()0,'transient retry',25) + compose('unpause','minio');minio_paused=False + wait(lambda:c.call('GET','/health/ready')['status']==200,'API restored') + c.wait_ready(alice,queued) + def retry_chain(): + d=trace_data(retry_trace) + return d if sum(span['name']=='queue.consume' for span in spans(d))>=2 else None + retry_data=wait(retry_chain,'retry chain') + (EVIDENCE/'lab6-retry-trace.json').write_text(json.dumps(retry_data,indent=2)) + record('retry_keeps_upload_trace',trace_id=retry_trace,consumer_spans=sum(span['name']=='queue.consume' for span in spans(retry_data))) + wait(lambda:value('max(queue_depth)')==0 and value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==0,'queue alert resolved') + record('incident_worker_resolved') + # Block INSERT while SELECT readiness remains available. This isolates + # DB latency from application CPU/HTTP handler latency. + api=compose('ps','-q','api').split()[0] + code='''import asyncio,asyncpg,sys +from src.core.settings import settings +async def main(): + c=await asyncpg.connect(settings.database_url.get_secret_value()) + await c.execute('BEGIN; LOCK TABLE messages IN SHARE MODE') + print('LOCKED',flush=True) + await asyncio.to_thread(sys.stdin.readline) + await c.execute('ROLLBACK');await c.close() +asyncio.run(main())''' + locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',code],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True) + check(locker.stdout.readline().strip()=='LOCKED','DB fault not active') + def failing(_): + return Client(c.base,timeout=12).call('POST','/api/v1/chats/'+chat['id']+'/messages',503, + actor=alice,body={'text':'bounded DB incident'},headers={'Idempotency-Key':uuid.uuid4().hex}) + with ThreadPoolExecutor(max_workers=4) as pool:errors=list(pool.map(failing,range(12))) + wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==1,'error alert firing',20) + db_trace=wait(lambda: trace_data(errors[0]['headers']['X-Trace-Id']),'DB trace') + failures=[span for span in spans(db_trace) if span['name'].startswith('db.') and span.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR')] + check(failures,'DB error span missing') + db_logs=wait(lambda:logs(event='db.error'),'DB error logs') + (EVIDENCE/'lab6-db-incident-trace.json').write_text(json.dumps(db_trace,indent=2)) + (EVIDENCE/'lab6-db-incident-logs.json').write_text(json.dumps(db_logs,indent=2)) + record('incident_DB_firing',errors=len(errors),db_error_spans=len(failures), + diagnosis='db.INSERT timeout, matching db.error event; API readiness remains available') + locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10);locker=None + c.message(alice,chat,'DB recovered') + wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==0,'error alert resolved',40) + record('incident_DB_resolved') + compose('stop','-t','2','tempo');tempo_stopped=True + started=time.monotonic() + for _ in range(15):c.message(alice,chat,'telemetry outage') + other=c.attachment(alice,chat)[0];c.wait_ready(alice,other) + # Bounded retry is 10s; wait only for an observed failure, not a benchmark. + wait(lambda:value('sum(otelcol_exporter_send_failed_spans_total)')>0 or value('sum(otelcol_exporter_send_failed_spans)')>0,'export failure counter',20) + usage=json.loads(compose('exec','-T','api','python','-c','from pathlib import Path;import json;print(json.dumps({"bytes":int(Path("/sys/fs/cgroup/memory.current").read_text()),"limit":int(Path("/sys/fs/cgroup/memory.max").read_text())}))').splitlines()[0]) + check(usage['bytes']